From d369915ed5ad7f8d743451d07817eee6964d9285 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Fri, 19 Dec 2025 17:18:35 +0800 Subject: [PATCH 01/59] [feat]hot standby for ha --- mooncake-store/include/hot_standby_service.h | 163 +++++++++++++ mooncake-store/include/master_service.h | 5 + mooncake-store/include/oplog_manager.h | 76 ++++++ mooncake-store/include/replication_service.h | 147 +++++++++++ mooncake-store/src/hot_standby_service.cpp | 243 +++++++++++++++++++ mooncake-store/src/master_service.cpp | 22 +- mooncake-store/src/oplog_manager.cpp | 95 ++++++++ mooncake-store/src/replication_service.cpp | 145 +++++++++++ 8 files changed, 895 insertions(+), 1 deletion(-) create mode 100644 mooncake-store/include/hot_standby_service.h create mode 100644 mooncake-store/include/oplog_manager.h create mode 100644 mooncake-store/include/replication_service.h create mode 100644 mooncake-store/src/hot_standby_service.cpp create mode 100644 mooncake-store/src/oplog_manager.cpp create mode 100644 mooncake-store/src/replication_service.cpp diff --git a/mooncake-store/include/hot_standby_service.h b/mooncake-store/include/hot_standby_service.h new file mode 100644 index 0000000000..5ba88c5007 --- /dev/null +++ b/mooncake-store/include/hot_standby_service.h @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "types.h" + +namespace mooncake { + +// Forward declarations +class MasterService; +class ReplicationStream; + +/** + * @brief Configuration for HotStandbyService + */ +struct HotStandbyConfig { + std::string standby_id; + std::string primary_address; + uint32_t replication_port{0}; + uint32_t verification_interval_sec{30}; + uint32_t max_replication_lag_entries{1000}; + bool enable_verification{true}; +}; + +/** + * @brief Sync status information for HotStandbyService + */ +struct StandbySyncStatus { + uint64_t applied_seq_id{0}; + uint64_t primary_seq_id{0}; + uint64_t lag_entries{0}; + std::chrono::milliseconds lag_time{0}; + bool is_syncing{false}; + bool is_connected{false}; +}; + +/** + * @brief HotStandbyService manages standby replication and promotion + * + * This service runs on Standby Master nodes and is responsible for: + * - Connecting to Primary and receiving OpLog entries + * - Applying OpLog entries to local metadata store + * - Periodically verifying data consistency with Primary + * - Promoting to Primary when elected as new Leader + * + * For now, this is a skeleton implementation without actual network + * communication. The gRPC integration will be added later. + */ +class HotStandbyService { + public: + explicit HotStandbyService(const HotStandbyConfig& config); + ~HotStandbyService(); + + /** + * @brief Start connecting to Primary and begin replication + * @param primary_address Address of the Primary Master + * @return ErrorCode::OK on success + */ + ErrorCode Start(const std::string& primary_address); + + /** + * @brief Stop replication and disconnect from Primary + */ + void Stop(); + + /** + * @brief Get current synchronization status + * @return StandbySyncStatus with current sync state + */ + StandbySyncStatus GetSyncStatus() const; + + /** + * @brief Check if standby is ready for promotion + * @return true if replication lag is within threshold + */ + bool IsReadyForPromotion() const; + + /** + * @brief Promote this standby to Primary + * + * This method should be called after successful leader election. + * It returns a MasterService instance initialized with the replicated + * metadata, ready to serve as the new Primary. + * + * @return Unique pointer to MasterService, or nullptr on failure + */ + std::unique_ptr Promote(); + + /** + * @brief Get the number of metadata entries in the local store + */ + size_t GetMetadataCount() const; + + private: + /** + * @brief Main replication loop (runs in background thread) + */ + void ReplicationLoop(); + + /** + * @brief Verification loop (runs in background thread) + */ + void VerificationLoop(); + + /** + * @brief Apply a single OpLog entry to local metadata store + * @param entry The OpLog entry to apply + */ + void ApplyOpLogEntry(const OpLogEntry& entry); + + /** + * @brief Connect to Primary and establish replication stream + * @return true on success, false on failure + */ + bool ConnectToPrimary(); + + /** + * @brief Disconnect from Primary + */ + void DisconnectFromPrimary(); + + /** + * @brief Process a batch of OpLog entries received from Primary + * @param entries Batch of OpLog entries + */ + void ProcessOpLogBatch(const std::vector& entries); + + HotStandbyConfig config_; + + // Metadata store (simplified - in full implementation this would be + // a complete replica of MasterService's metadata) + // For now, we use a placeholder structure + struct MetadataStore { + // Placeholder: In full implementation, this would mirror + // MasterService's metadata_shards_ structure + size_t entry_count{0}; + }; + std::unique_ptr metadata_store_; + + // Replication state + std::shared_ptr replication_stream_; + std::atomic applied_seq_id_{0}; + std::atomic primary_seq_id_{0}; + std::atomic running_{false}; + std::atomic is_connected_{false}; + + // Background threads + std::thread replication_thread_; + std::thread verification_thread_; + + // Synchronization + mutable std::mutex mutex_; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 35e728f0c5..4824ff48d3 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -25,6 +25,7 @@ #include "master_config.h" #include "rpc_types.h" #include "replica.h" +#include "oplog_manager.h" namespace mooncake { // Forward declarations @@ -631,6 +632,10 @@ class MasterService { // Segment management SegmentManager segment_manager_; BufferAllocatorType memory_allocator_type_; + + // Operation log manager for hot-standby replication. It records + // state-changing operations so that a standby master can replay them. + OpLogManager oplog_manager_; std::shared_ptr allocation_strategy_; // Discarded replicas management diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h new file mode 100644 index 0000000000..1d9e26ded5 --- /dev/null +++ b/mooncake-store/include/oplog_manager.h @@ -0,0 +1,76 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace mooncake { + +// Operation types for hot-standby replication. +// This is a minimal subset that can be extended later. +enum class OpType : uint8_t { + PUT_END = 1, + PUT_REVOKE = 2, + REMOVE = 3, + LEASE_RENEW = 4, +}; + +// A single operation log entry. +struct OpLogEntry { + uint64_t sequence_id{0}; // Monotonically increasing sequence + uint64_t timestamp_ms{0}; // Logical timestamp in milliseconds + OpType op_type{OpType::PUT_END}; + std::string object_key; // Target object key + std::string payload; // Serialized extra data (optional) + uint32_t checksum{0}; // Checksum of payload (implementation-defined) + uint32_t prefix_hash{0}; // Hash of key prefix (for future verification) +}; + +/** + * @brief In-memory operation log manager. + * + * This class is intentionally simple: it keeps a bounded deque of OpLogEntry + * and provides append / get-since primitives. It can later be extended to + * notify ReplicationService or to spill to disk if needed. + */ +class OpLogManager { + public: + OpLogManager(); + + // Append a new entry and return the assigned sequence_id. + uint64_t Append(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Get entries with sequence_id > since_seq_id, up to at most limit entries. + std::vector GetEntriesSince(uint64_t since_seq_id, + size_t limit = 1000) const; + + // Get the latest assigned sequence id. Returns 0 if no entry exists. + uint64_t GetLastSequenceId() const; + + // Truncate all entries with sequence_id < min_seq_to_keep. + void TruncateBefore(uint64_t min_seq_to_keep); + + // Current number of entries in the buffer. + size_t GetEntryCount() const; + + private: + static uint64_t NowMs(); + static uint32_t ComputeChecksum(const std::string& data); + static uint32_t ComputePrefixHash(const std::string& key); + + mutable std::shared_mutex mutex_; + std::deque buffer_; + uint64_t first_seq_id_{1}; // sequence_id of buffer_.front() + uint64_t last_seq_id_{0}; // last assigned sequence_id + + // Simple bounds to avoid unbounded memory growth. + static constexpr size_t kMaxBufferEntries_ = 100000; +}; + +} // namespace mooncake + + diff --git a/mooncake-store/include/replication_service.h b/mooncake-store/include/replication_service.h new file mode 100644 index 0000000000..60a9abf5c7 --- /dev/null +++ b/mooncake-store/include/replication_service.h @@ -0,0 +1,147 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "types.h" + +namespace mooncake { + +// Forward declarations +class MasterService; +class OpLogManager; + +/** + * @brief Replication stream interface (placeholder for future gRPC implementation) + * + * This is a minimal interface that will be replaced with actual gRPC streaming + * in the future. For now, it serves as a placeholder to establish the + * architecture. + */ +class ReplicationStream { + public: + virtual ~ReplicationStream() = default; + virtual bool Send(const std::vector& entries) = 0; + virtual bool IsConnected() const = 0; +}; + +/** + * @brief Verification request/response structures + */ +struct VerificationRequest { + std::string standby_id; + std::vector> samples; // (key, checksum) + uint32_t prefix_hash; +}; + +struct VerificationResponse { + std::vector mismatched_keys; // Keys with checksum mismatch + bool is_consistent; +}; + +/** + * @brief ReplicationService manages OpLog replication from Primary to Standbys + * + * This service runs on the Primary Master and is responsible for: + * - Broadcasting OpLog entries to all connected Standby nodes + * - Tracking replication lag for each Standby + * - Handling verification requests from Standbys + * + * For now, this is a skeleton implementation without actual network + * communication. The gRPC integration will be added later. + */ +class ReplicationService { + public: + explicit ReplicationService(OpLogManager& oplog_manager, + MasterService& master_service); + + ~ReplicationService(); + + /** + * @brief Register a new Standby connection + * @param standby_id Unique identifier for the Standby node + * @param stream Replication stream for sending OpLog entries + */ + void RegisterStandby(const std::string& standby_id, + std::shared_ptr stream); + + /** + * @brief Unregister a Standby (when it disconnects) + * @param standby_id Unique identifier for the Standby node + */ + void UnregisterStandby(const std::string& standby_id); + + /** + * @brief Called by OpLogManager when a new entry is appended + * @param entry The newly appended OpLog entry + * + * This method should be called by OpLogManager (via callback) or + * directly from MasterService after appending to OpLog. + */ + void OnNewOpLog(const OpLogEntry& entry); + + /** + * @brief Handle verification request from a Standby + * @param request Verification request containing checksums + * @return Verification response with mismatched keys + */ + VerificationResponse HandleVerification(const VerificationRequest& request); + + /** + * @brief Get replication lag for each Standby + * @return Map of standby_id -> lag in sequence IDs + */ + std::map GetReplicationLag() const; + + /** + * @brief Get the number of connected Standbys + */ + size_t GetStandbyCount() const; + + private: + /** + * @brief Broadcast an OpLog entry to all connected Standbys + * @param entry The OpLog entry to broadcast + */ + void BroadcastEntry(const OpLogEntry& entry); + + /** + * @brief Send a batch of OpLog entries to a specific Standby + * @param standby_id Target Standby identifier + * @param entries Batch of OpLog entries to send + */ + void SendBatch(const std::string& standby_id, + const std::vector& entries); + + /** + * @brief State for each connected Standby + */ + struct StandbyState { + std::shared_ptr stream; + uint64_t acked_seq_id{0}; // Last acknowledged sequence ID + std::chrono::steady_clock::time_point last_ack_time; + std::vector pending_batch; // Batched entries + }; + + OpLogManager& oplog_manager_; + MasterService& master_service_; + + mutable std::shared_mutex mutex_; + std::unordered_map standbys_; + + // Batch configuration + static constexpr size_t kBatchSize = 100; + static constexpr uint32_t kBatchTimeoutMs = 10; +}; + +} // namespace mooncake + diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp new file mode 100644 index 0000000000..6e6e973173 --- /dev/null +++ b/mooncake-store/src/hot_standby_service.cpp @@ -0,0 +1,243 @@ +#include "hot_standby_service.h" + +#include + +#include +#include + +#include "master_service.h" +#include "oplog_manager.h" + +namespace mooncake { + +HotStandbyService::HotStandbyService(const HotStandbyConfig& config) + : config_(config) { + metadata_store_ = std::make_unique(); +} + +HotStandbyService::~HotStandbyService() { + Stop(); +} + +ErrorCode HotStandbyService::Start(const std::string& primary_address) { + std::lock_guard lock(mutex_); + + if (running_.load()) { + LOG(WARNING) << "HotStandbyService is already running"; + return ErrorCode::OK; + } + + config_.primary_address = primary_address; + running_.store(true); + + // Start background threads + replication_thread_ = std::thread(&HotStandbyService::ReplicationLoop, this); + if (config_.enable_verification) { + verification_thread_ = + std::thread(&HotStandbyService::VerificationLoop, this); + } + + LOG(INFO) << "HotStandbyService started, connecting to Primary: " + << primary_address; + return ErrorCode::OK; +} + +void HotStandbyService::Stop() { + if (!running_.load()) { + return; + } + + running_.store(false); + DisconnectFromPrimary(); + + // Wait for threads to finish + if (replication_thread_.joinable()) { + replication_thread_.join(); + } + if (verification_thread_.joinable()) { + verification_thread_.join(); + } + + LOG(INFO) << "HotStandbyService stopped"; +} + +StandbySyncStatus HotStandbyService::GetSyncStatus() const { + StandbySyncStatus status; + status.applied_seq_id = applied_seq_id_.load(); + status.primary_seq_id = primary_seq_id_.load(); + status.is_connected = is_connected_.load(); + + if (status.primary_seq_id > status.applied_seq_id) { + status.lag_entries = status.primary_seq_id - status.applied_seq_id; + } else { + status.lag_entries = 0; + } + + // Calculate lag time (placeholder - in full implementation this would + // track actual time differences) + status.lag_time = std::chrono::milliseconds(0); + status.is_syncing = running_.load() && is_connected_.load(); + + return status; +} + +bool HotStandbyService::IsReadyForPromotion() const { + StandbySyncStatus status = GetSyncStatus(); + if (!status.is_connected) { + return false; + } + + // Check if lag is within threshold + return status.lag_entries <= config_.max_replication_lag_entries; +} + +std::unique_ptr HotStandbyService::Promote() { + std::lock_guard lock(mutex_); + + if (!IsReadyForPromotion()) { + LOG(ERROR) << "Standby is not ready for promotion. Lag: " + << GetSyncStatus().lag_entries << " entries"; + return nullptr; + } + + LOG(INFO) << "Promoting Standby to Primary. Applied seq_id: " + << applied_seq_id_.load(); + + // Stop replication + Stop(); + + // In full implementation, we would: + // 1. Create a new MasterService instance + // 2. Initialize it with the replicated metadata from metadata_store_ + // 3. Return the MasterService instance + + // For now, this is a placeholder + // TODO: Implement full promotion logic + auto master_service = std::make_unique(); + + LOG(INFO) << "Standby promoted to Primary successfully"; + return master_service; +} + +size_t HotStandbyService::GetMetadataCount() const { + std::lock_guard lock(mutex_); + return metadata_store_ ? metadata_store_->entry_count : 0; +} + +void HotStandbyService::ReplicationLoop() { + LOG(INFO) << "Replication loop started"; + + while (running_.load()) { + // Try to connect if not connected + if (!is_connected_.load()) { + if (ConnectToPrimary()) { + is_connected_.store(true); + LOG(INFO) << "Connected to Primary: " << config_.primary_address; + } else { + // Retry after a delay + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + } + + // In full implementation, this would: + // 1. Receive OpLog entries from Primary via gRPC stream + // 2. Process them in batches + // 3. Apply to local metadata store + + // For now, this is a placeholder that simulates receiving entries + // In the actual implementation, this would block on the gRPC stream + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Placeholder: Simulate receiving entries + // TODO: Replace with actual gRPC stream reading + } + + LOG(INFO) << "Replication loop stopped"; +} + +void HotStandbyService::VerificationLoop() { + LOG(INFO) << "Verification loop started"; + + while (running_.load()) { + std::this_thread::sleep_for( + std::chrono::seconds(config_.verification_interval_sec)); + + if (!is_connected_.load()) { + continue; + } + + // In full implementation, this would: + // 1. Sample keys from local metadata store + // 2. Calculate checksums + // 3. Send verification request to Primary + // 4. Handle mismatches if any + + // Placeholder: Log that verification would happen + VLOG(1) << "Verification check (placeholder)"; + } + + LOG(INFO) << "Verification loop stopped"; +} + +void HotStandbyService::ApplyOpLogEntry(const OpLogEntry& entry) { + // In full implementation, this would apply the OpLog entry to + // the local metadata store, mirroring the operations in MasterService. + + // For now, this is a placeholder that just updates counters + switch (entry.op_type) { + case OpType::PUT_END: + // Create or update metadata for the key + if (metadata_store_) { + metadata_store_->entry_count++; + } + break; + case OpType::PUT_REVOKE: + case OpType::REMOVE: + // Remove metadata for the key + if (metadata_store_ && metadata_store_->entry_count > 0) { + metadata_store_->entry_count--; + } + break; + case OpType::LEASE_RENEW: + // Update lease timeout (no change to entry count) + break; + default: + LOG(WARNING) << "Unknown OpType: " + << static_cast(entry.op_type); + break; + } + + applied_seq_id_.store(entry.sequence_id); +} + +void HotStandbyService::ProcessOpLogBatch( + const std::vector& entries) { + for (const auto& entry : entries) { + ApplyOpLogEntry(entry); + } +} + +bool HotStandbyService::ConnectToPrimary() { + // In full implementation, this would: + // 1. Create a gRPC channel to Primary + // 2. Establish a bidirectional stream for OpLog replication + // 3. Send initial sync request with current applied_seq_id + // 4. Start receiving OpLog entries + + // For now, this is a placeholder + LOG(INFO) << "Connecting to Primary: " << config_.primary_address + << " (placeholder)"; + return false; // Return false to indicate not yet implemented +} + +void HotStandbyService::DisconnectFromPrimary() { + if (is_connected_.load()) { + is_connected_.store(false); + replication_stream_.reset(); + LOG(INFO) << "Disconnected from Primary"; + } +} + +} // namespace mooncake + diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index e049cb10eb..cb96878525 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -2,9 +2,9 @@ #include #include -#include #include #include +#include #include #include "master_metric_manager.h" @@ -231,6 +231,8 @@ auto MasterService::ExistKey(const std::string& key) // client. metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Record lease renewal for standby synchronization. + oplog_manager_.Append(OpType::LEASE_RENEW, key); return true; } } @@ -501,6 +503,8 @@ auto MasterService::GetReplicaListByRegex(const std::string& regex_pattern) results.emplace(key, std::move(replica_list)); metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Record lease renewal for standby synchronization. + oplog_manager_.Append(OpType::LEASE_RENEW, key); } } } @@ -542,6 +546,8 @@ auto MasterService::GetReplicaList(std::string_view key) // Grant a lease to the object so it will not be removed // when the client is reading it. metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Record lease renewal for standby synchronization. + oplog_manager_.Append(OpType::LEASE_RENEW, std::string(key)); return GetReplicaListResponse(std::move(replica_list), default_kv_lease_ttl_); @@ -696,6 +702,12 @@ auto MasterService::PutEnd(const UUID& client_id, const std::string& key, // at beginning. 2. If this object has soft pin enabled, set it to be soft // pinned. metadata.GrantLease(0, default_kv_soft_pin_ttl_); + + // Record OpLog entry for PUT_END so that standbys can replay this change. + // For now we do not include extra payload; it can be extended later if + // needed (e.g. to carry replica descriptors). + oplog_manager_.Append(OpType::PUT_END, key); + return {}; } @@ -776,6 +788,10 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, if (metadata.IsValid() == false) { accessor.Erase(); } + + // Log the revoke operation so that standbys can roll back their metadata. + oplog_manager_.Append(OpType::PUT_REVOKE, key); + return {}; } @@ -821,6 +837,10 @@ auto MasterService::Remove(const std::string& key) // Remove object metadata accessor.Erase(); + + // Log explicit remove so that standbys can delete the same key. + oplog_manager_.Append(OpType::REMOVE, key); + return {}; } diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp new file mode 100644 index 0000000000..5dbc447bc0 --- /dev/null +++ b/mooncake-store/src/oplog_manager.cpp @@ -0,0 +1,95 @@ +#include "oplog_manager.h" + +#include +#include +#include + +namespace mooncake { + +OpLogManager::OpLogManager() = default; + +uint64_t OpLogManager::Append(OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry entry; + entry.op_type = type; + entry.object_key = key; + entry.payload = payload; + entry.timestamp_ms = NowMs(); + entry.checksum = ComputeChecksum(entry.payload); + entry.prefix_hash = ComputePrefixHash(entry.object_key); + + std::unique_lock lock(mutex_); + entry.sequence_id = ++last_seq_id_; + + if (buffer_.size() >= kMaxBufferEntries_) { + buffer_.pop_front(); + ++first_seq_id_; + } + + buffer_.emplace_back(std::move(entry)); + return last_seq_id_; +} + +std::vector OpLogManager::GetEntriesSince(uint64_t since_seq_id, + size_t limit) const { + std::shared_lock lock(mutex_); + std::vector result; + if (buffer_.empty() || since_seq_id >= last_seq_id_) { + return result; + } + + result.reserve(std::min(limit, buffer_.size())); + for (const auto& e : buffer_) { + if (e.sequence_id > since_seq_id) { + result.push_back(e); + if (result.size() >= limit) { + break; + } + } + } + return result; +} + +uint64_t OpLogManager::GetLastSequenceId() const { + std::shared_lock lock(mutex_); + return last_seq_id_; +} + +void OpLogManager::TruncateBefore(uint64_t min_seq_to_keep) { + std::unique_lock lock(mutex_); + while (!buffer_.empty() && buffer_.front().sequence_id < min_seq_to_keep) { + buffer_.pop_front(); + ++first_seq_id_; + } +} + +size_t OpLogManager::GetEntryCount() const { + std::shared_lock lock(mutex_); + return buffer_.size(); +} + +uint64_t OpLogManager::NowMs() { + using namespace std::chrono; + return duration_cast(steady_clock::now().time_since_epoch()) + .count(); +} + +uint32_t OpLogManager::ComputeChecksum(const std::string& data) { + // NOTE: For now we use a simple hash as a placeholder. This can be + // replaced with a real CRC32 implementation later if needed. + return static_cast(std::hash{}(data)); +} + +uint32_t OpLogManager::ComputePrefixHash(const std::string& key) { + if (key.empty()) { + return 0; + } + // Use at most first 8 characters to compute a simple hash. + const size_t prefix_len = std::min(8, key.size()); + return static_cast( + std::hash{}(std::string_view(key.data(), prefix_len))); +} + +} // namespace mooncake + + diff --git a/mooncake-store/src/replication_service.cpp b/mooncake-store/src/replication_service.cpp new file mode 100644 index 0000000000..ad8e938f2d --- /dev/null +++ b/mooncake-store/src/replication_service.cpp @@ -0,0 +1,145 @@ +#include "replication_service.h" + +#include + +#include +#include + +#include "master_service.h" +#include "oplog_manager.h" + +namespace mooncake { + +ReplicationService::ReplicationService(OpLogManager& oplog_manager, + MasterService& master_service) + : oplog_manager_(oplog_manager), master_service_(master_service) {} + +ReplicationService::~ReplicationService() { + std::unique_lock lock(mutex_); + standbys_.clear(); +} + +void ReplicationService::RegisterStandby(const std::string& standby_id, + std::shared_ptr stream) { + std::unique_lock lock(mutex_); + + StandbyState state; + state.stream = std::move(stream); + state.acked_seq_id = 0; + state.last_ack_time = std::chrono::steady_clock::now(); + + standbys_[standby_id] = std::move(state); + + LOG(INFO) << "Registered Standby: " << standby_id + << ", total standbys: " << standbys_.size(); +} + +void ReplicationService::UnregisterStandby(const std::string& standby_id) { + std::unique_lock lock(mutex_); + + auto it = standbys_.find(standby_id); + if (it != standbys_.end()) { + standbys_.erase(it); + LOG(INFO) << "Unregistered Standby: " << standby_id + << ", remaining standbys: " << standbys_.size(); + } +} + +void ReplicationService::OnNewOpLog(const OpLogEntry& entry) { + std::shared_lock lock(mutex_); + + if (standbys_.empty()) { + // No standbys connected, nothing to do + return; + } + + // Broadcast to all standbys + BroadcastEntry(entry); +} + +void ReplicationService::BroadcastEntry(const OpLogEntry& entry) { + // For now, we just add the entry to each standby's pending batch. + // In a full implementation, this would trigger immediate or batched sending. + for (auto& [standby_id, state] : standbys_) { + state.pending_batch.push_back(entry); + + // If batch is full, send it immediately + if (state.pending_batch.size() >= kBatchSize) { + SendBatch(standby_id, state.pending_batch); + state.pending_batch.clear(); + } + } +} + +void ReplicationService::SendBatch(const std::string& standby_id, + const std::vector& entries) { + auto it = standbys_.find(standby_id); + if (it == standbys_.end()) { + LOG(WARNING) << "Attempted to send batch to unknown Standby: " + << standby_id; + return; + } + + auto& state = it->second; + if (!state.stream || !state.stream->IsConnected()) { + LOG(WARNING) << "Standby stream not connected: " << standby_id; + return; + } + + // For now, this is a placeholder. In the full implementation, + // this would send via gRPC stream. + bool success = state.stream->Send(entries); + if (success) { + // Update ACK tracking (in full implementation, this would be + // updated when we receive actual ACK from Standby) + if (!entries.empty()) { + state.acked_seq_id = entries.back().sequence_id; + state.last_ack_time = std::chrono::steady_clock::now(); + } + } else { + LOG(ERROR) << "Failed to send batch to Standby: " << standby_id; + } +} + +VerificationResponse ReplicationService::HandleVerification( + const VerificationRequest& request) { + VerificationResponse response; + response.is_consistent = true; + + // TODO: Implement actual verification logic by comparing checksums + // with MasterService metadata. For now, this is a placeholder. + + LOG(INFO) << "Verification request from Standby: " << request.standby_id + << ", samples: " << request.samples.size(); + + // Placeholder: assume consistent for now + response.is_consistent = true; + response.mismatched_keys.clear(); + + return response; +} + +std::map ReplicationService::GetReplicationLag() const { + std::shared_lock lock(mutex_); + std::map lag_map; + + uint64_t primary_seq_id = oplog_manager_.GetLastSequenceId(); + + for (const auto& [standby_id, state] : standbys_) { + uint64_t lag = 0; + if (primary_seq_id > state.acked_seq_id) { + lag = primary_seq_id - state.acked_seq_id; + } + lag_map[standby_id] = lag; + } + + return lag_map; +} + +size_t ReplicationService::GetStandbyCount() const { + std::shared_lock lock(mutex_); + return standbys_.size(); +} + +} // namespace mooncake + From cb96c1978d6ad6e517c33326f75b387adfb2974b Mon Sep 17 00:00:00 2001 From: BernardLee Date: Wed, 24 Dec 2025 17:08:40 +0800 Subject: [PATCH 02/59] ha --- mooncake-store/include/master_service.h | 17 ++++++++++++ mooncake-store/include/rpc_service.h | 14 ++++++++++ mooncake-store/src/CMakeLists.txt | 12 +++++++++ mooncake-store/src/master_service.cpp | 36 ++++++++++++++++++++----- mooncake-store/src/oplog_manager.cpp | 7 ++--- mooncake-store/src/rpc_service.cpp | 10 +++++++ 6 files changed, 87 insertions(+), 9 deletions(-) diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 4824ff48d3..3d4cf1c915 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -31,6 +31,7 @@ namespace mooncake { // Forward declarations class AllocationStrategy; class EvictionStrategy; +class ReplicationService; /* * @brief MasterService is the main class for the master server. @@ -272,6 +273,18 @@ class MasterService { */ tl::expected GetStorageConfig() const; + /** + * @brief Get OpLogManager reference for external access (e.g., ReplicationService) + * @return Reference to the OpLogManager instance + */ + OpLogManager& GetOpLogManager(); + + /** + * @brief Set ReplicationService pointer for OpLog notification + * @param replication_service Pointer to ReplicationService (can be nullptr) + */ + void SetReplicationService(ReplicationService* replication_service); + /** * @brief Mounts a file storage segment into the master. * @param enable_offloading If true, enables offloading (write-to-file). @@ -636,6 +649,10 @@ class MasterService { // Operation log manager for hot-standby replication. It records // state-changing operations so that a standby master can replay them. OpLogManager oplog_manager_; + + // ReplicationService pointer for OpLog notification (set by WrappedMasterService) + ReplicationService* replication_service_{nullptr}; + std::shared_ptr allocation_strategy_; // Discarded replicas management diff --git a/mooncake-store/include/rpc_service.h b/mooncake-store/include/rpc_service.h index 1cf5d7a24f..1013398d8c 100644 --- a/mooncake-store/include/rpc_service.h +++ b/mooncake-store/include/rpc_service.h @@ -13,6 +13,11 @@ #include "rpc_types.h" #include "master_config.h" +// Forward declaration +namespace mooncake { +class ReplicationService; +} + namespace mooncake { extern const uint64_t kMetricReportIntervalSeconds; @@ -109,11 +114,20 @@ class WrappedMasterService { const UUID& client_id, const std::vector& keys, const std::vector& metadatas); + /** + * @brief Get ReplicationService pointer for RPC layer access + * @return Pointer to ReplicationService, or nullptr if not initialized + */ + ReplicationService* GetReplicationService(); + private: MasterService master_service_; std::thread metric_report_thread_; coro_http::coro_http_server http_server_; std::atomic metric_report_running_; + + // ReplicationService for hot-standby replication (only initialized in HA mode) + std::unique_ptr replication_service_; }; void RegisterRpcService(coro_rpc::coro_rpc_server& server, diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 05968b56df..287e585bd5 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -30,6 +30,16 @@ set(MOONCAKE_STORE_SOURCES set(EXTRA_LIBS "") +# Find xxHash (required for ComputeChecksum) +find_path(XXHASH_INCLUDE_DIR NAMES xxhash.h PATHS /usr/include /usr/local/include) +find_library(XXHASH_LIBRARY NAMES xxhash libxxhash PATHS /usr/lib /usr/local/lib /usr/lib64) +if (XXHASH_INCLUDE_DIR AND XXHASH_LIBRARY) + message(STATUS "Found xxHash: include=${XXHASH_INCLUDE_DIR} lib=${XXHASH_LIBRARY}") + list(APPEND MASTER_EXTRA_INCS ${XXHASH_INCLUDE_DIR}) +else() + message(FATAL_ERROR "xxHash library/header not found. Please install xxhash (development headers) and try again.") +endif() + if(USE_3FS) add_subdirectory(hf3fs) list(APPEND MOONCAKE_STORE_SOURCES ${HF3FS_SOURCES}) @@ -43,6 +53,8 @@ endif() # The cache_allocator library include_directories(${Python3_INCLUDE_DIRS}) add_library(mooncake_store ${MOONCAKE_STORE_SOURCES}) +target_include_directories(mooncake_store PUBLIC ${XXHASH_INCLUDE_DIR}) +target_link_libraries(mooncake_store PUBLIC ${XXHASH_LIBRARY}) target_link_libraries(mooncake_store PUBLIC transfer_engine cachelib_memory_allocator ${ETCD_WRAPPER_LIB} glog::glog gflags::gflags ${EXTRA_LIBS} ) diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index cb96878525..2ddd594795 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -10,6 +10,7 @@ #include "master_metric_manager.h" #include "segment.h" #include "types.h" +#include "replication_service.h" namespace mooncake { @@ -74,6 +75,21 @@ MasterService::MasterService(const MasterServiceConfig& config) } } +// Helper function to append OpLog entry and notify ReplicationService +void MasterService::AppendOpLogAndNotify(OpType type, const std::string& key, + const std::string& payload) { + uint64_t seq_id = oplog_manager_.Append(type, key, payload); + + // Notify ReplicationService if it's set + if (replication_service_ != nullptr) { + // Get the entry we just appended (GetEntriesSince returns entries with seq_id > since_seq_id) + auto entries = oplog_manager_.GetEntriesSince(seq_id - 1, 1); + if (!entries.empty() && entries[0].sequence_id == seq_id) { + replication_service_->OnNewOpLog(entries[0]); + } + } +} + MasterService::~MasterService() { // Stop and join the threads eviction_running_ = false; @@ -232,7 +248,7 @@ auto MasterService::ExistKey(const std::string& key) metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); // Record lease renewal for standby synchronization. - oplog_manager_.Append(OpType::LEASE_RENEW, key); + AppendOpLogAndNotify(OpType::LEASE_RENEW, key); return true; } } @@ -504,7 +520,7 @@ auto MasterService::GetReplicaListByRegex(const std::string& regex_pattern) metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); // Record lease renewal for standby synchronization. - oplog_manager_.Append(OpType::LEASE_RENEW, key); + AppendOpLogAndNotify(OpType::LEASE_RENEW, key); } } } @@ -547,7 +563,7 @@ auto MasterService::GetReplicaList(std::string_view key) // when the client is reading it. metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); // Record lease renewal for standby synchronization. - oplog_manager_.Append(OpType::LEASE_RENEW, std::string(key)); + AppendOpLogAndNotify(OpType::LEASE_RENEW, std::string(key)); return GetReplicaListResponse(std::move(replica_list), default_kv_lease_ttl_); @@ -706,7 +722,7 @@ auto MasterService::PutEnd(const UUID& client_id, const std::string& key, // Record OpLog entry for PUT_END so that standbys can replay this change. // For now we do not include extra payload; it can be extended later if // needed (e.g. to carry replica descriptors). - oplog_manager_.Append(OpType::PUT_END, key); + AppendOpLogAndNotify(OpType::PUT_END, key); return {}; } @@ -790,7 +806,7 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, } // Log the revoke operation so that standbys can roll back their metadata. - oplog_manager_.Append(OpType::PUT_REVOKE, key); + AppendOpLogAndNotify(OpType::PUT_REVOKE, key); return {}; } @@ -839,7 +855,7 @@ auto MasterService::Remove(const std::string& key) accessor.Erase(); // Log explicit remove so that standbys can delete the same key. - oplog_manager_.Append(OpType::REMOVE, key); + AppendOpLogAndNotify(OpType::REMOVE, key); return {}; } @@ -1579,4 +1595,12 @@ std::string MasterService::ResolvePath(const std::string& key) const { return full_path.lexically_normal().string(); } +OpLogManager& MasterService::GetOpLogManager() { + return oplog_manager_; +} + +void MasterService::SetReplicationService(ReplicationService* replication_service) { + replication_service_ = replication_service; +} + } // namespace mooncake \ No newline at end of file diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index 5dbc447bc0..54727e51d5 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace mooncake { @@ -75,9 +76,9 @@ uint64_t OpLogManager::NowMs() { } uint32_t OpLogManager::ComputeChecksum(const std::string& data) { - // NOTE: For now we use a simple hash as a placeholder. This can be - // replaced with a real CRC32 implementation later if needed. - return static_cast(std::hash{}(data)); + // Use xxHash XXH32 for a fast, deterministic 32-bit checksum. + // Requires linking against xxHash (e.g., libxxhash) and including . + return static_cast(XXH32(data.data(), data.size(), 0)); } uint32_t OpLogManager::ComputePrefixHash(const std::string& key) { diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index db1cec0cad..c9e1957a81 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -19,6 +19,7 @@ #include "types.h" #include "utils/scoped_vlog_timer.h" #include "version.h" +#include "replication_service.h" namespace mooncake { @@ -31,6 +32,15 @@ WrappedMasterService::WrappedMasterService( metric_report_running_(config.enable_metric_reporting) { init_http_server(); + // Initialize ReplicationService if HA mode is enabled + if (config.enable_ha) { + replication_service_ = std::make_unique( + master_service_.GetOpLogManager(), master_service_); + // Set ReplicationService pointer in MasterService for OpLog notification + master_service_.SetReplicationService(replication_service_.get()); + LOG(INFO) << "ReplicationService initialized for HA mode"; + } + if (config.enable_metric_reporting) { metric_report_thread_ = std::thread([this]() { while (metric_report_running_) { From 3a687684d6d4c09f9df9a7dbb7278d65c8adef1a Mon Sep 17 00:00:00 2001 From: BernardLee Date: Wed, 24 Dec 2025 17:12:49 +0800 Subject: [PATCH 03/59] fix --- mooncake-store/include/master_service.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 3d4cf1c915..412ec4001d 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -315,6 +315,15 @@ class MasterService { -> tl::expected; private: + /** + * @brief Helper function to append OpLog entry and notify ReplicationService + * @param type Operation type + * @param key Object key + * @param payload Optional payload data + */ + void AppendOpLogAndNotify(OpType type, const std::string& key, + const std::string& payload = std::string()); + // Resolve the key to a sanitized format for storage std::string SanitizeKey(const std::string& key) const; std::string ResolvePath(const std::string& key) const; From db782bbbdfabcafa801897bf1149a2fb7c5359f5 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Wed, 24 Dec 2025 17:17:06 +0800 Subject: [PATCH 04/59] fix --- mooncake-store/src/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 287e585bd5..a20b889981 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -26,6 +26,8 @@ set(MOONCAKE_STORE_SOURCES dummy_client.cpp http_metadata_server.cpp file_storage.cpp + oplog_manager.cpp + replication_service.cpp ) set(EXTRA_LIBS "") From 6a90b32c77915ae4c0ab684b544f2923c359c39c Mon Sep 17 00:00:00 2001 From: BernardLee Date: Wed, 24 Dec 2025 17:21:47 +0800 Subject: [PATCH 05/59] Ack --- mooncake-store/include/replication_service.h | 31 ++++++ mooncake-store/src/replication_service.cpp | 111 +++++++++++++++++-- 2 files changed, 134 insertions(+), 8 deletions(-) diff --git a/mooncake-store/include/replication_service.h b/mooncake-store/include/replication_service.h index 60a9abf5c7..2018fa06f9 100644 --- a/mooncake-store/include/replication_service.h +++ b/mooncake-store/include/replication_service.h @@ -107,6 +107,19 @@ class ReplicationService { */ size_t GetStandbyCount() const; + /** + * @brief Handle ACK from Standby + * @param standby_id Standby identifier + * @param acked_seq_id Acknowledged sequence ID + */ + void OnAck(const std::string& standby_id, uint64_t acked_seq_id); + + /** + * @brief Check health status of all Standbys and update their states + * This should be called periodically (e.g., by a background thread) + */ + void CheckStandbyHealth(); + private: /** * @brief Broadcast an OpLog entry to all connected Standbys @@ -125,11 +138,23 @@ class ReplicationService { /** * @brief State for each connected Standby */ + enum class StandbyHealthState { + HEALTHY, // Normal state, responding to requests + SLOW, // Responding slowly, but still processing + TIMEOUT, // Timeout detected, possibly failed + DISCONNECTED // Connection lost + }; + struct StandbyState { std::shared_ptr stream; uint64_t acked_seq_id{0}; // Last acknowledged sequence ID std::chrono::steady_clock::time_point last_ack_time; + std::chrono::steady_clock::time_point last_send_time; // Last send attempt time std::vector pending_batch; // Batched entries + + // Health monitoring + StandbyHealthState state{StandbyHealthState::HEALTHY}; + uint32_t consecutive_failures{0}; // Consecutive failure count }; OpLogManager& oplog_manager_; @@ -141,6 +166,12 @@ class ReplicationService { // Batch configuration static constexpr size_t kBatchSize = 100; static constexpr uint32_t kBatchTimeoutMs = 10; + + // Health check configuration + static constexpr uint32_t kAckTimeoutMs = 5000; // ACK timeout (5 seconds) + static constexpr uint32_t kSendTimeoutMs = 3000; // Send timeout (3 seconds) + static constexpr uint32_t kMaxConsecutiveFailures = 3; // Max consecutive failures before marking as TIMEOUT + static constexpr uint32_t kHealthCheckIntervalMs = 1000; // Health check interval (1 second) }; } // namespace mooncake diff --git a/mooncake-store/src/replication_service.cpp b/mooncake-store/src/replication_service.cpp index ad8e938f2d..6dfe7e4f0a 100644 --- a/mooncake-store/src/replication_service.cpp +++ b/mooncake-store/src/replication_service.cpp @@ -27,6 +27,9 @@ void ReplicationService::RegisterStandby(const std::string& standby_id, state.stream = std::move(stream); state.acked_seq_id = 0; state.last_ack_time = std::chrono::steady_clock::now(); + state.last_send_time = std::chrono::steady_clock::now(); + state.state = StandbyHealthState::HEALTHY; + state.consecutive_failures = 0; standbys_[standby_id] = std::move(state); @@ -58,9 +61,14 @@ void ReplicationService::OnNewOpLog(const OpLogEntry& entry) { } void ReplicationService::BroadcastEntry(const OpLogEntry& entry) { - // For now, we just add the entry to each standby's pending batch. - // In a full implementation, this would trigger immediate or batched sending. + // Skip Standbys that are in TIMEOUT or DISCONNECTED state for (auto& [standby_id, state] : standbys_) { + // Skip unhealthy Standbys + if (state.state == StandbyHealthState::TIMEOUT || + state.state == StandbyHealthState::DISCONNECTED) { + continue; + } + state.pending_batch.push_back(entry); // If batch is full, send it immediately @@ -81,23 +89,41 @@ void ReplicationService::SendBatch(const std::string& standby_id, } auto& state = it->second; + + // Check connection status if (!state.stream || !state.stream->IsConnected()) { + state.state = StandbyHealthState::DISCONNECTED; LOG(WARNING) << "Standby stream not connected: " << standby_id; return; } + // Update last send time + state.last_send_time = std::chrono::steady_clock::now(); + // For now, this is a placeholder. In the full implementation, // this would send via gRPC stream. bool success = state.stream->Send(entries); if (success) { - // Update ACK tracking (in full implementation, this would be - // updated when we receive actual ACK from Standby) - if (!entries.empty()) { - state.acked_seq_id = entries.back().sequence_id; - state.last_ack_time = std::chrono::steady_clock::now(); + // Reset failure count on successful send + // Note: ACK will be updated via OnAck() when we receive actual ACK + state.consecutive_failures = 0; + if (state.state == StandbyHealthState::SLOW) { + state.state = StandbyHealthState::HEALTHY; } } else { - LOG(ERROR) << "Failed to send batch to Standby: " << standby_id; + state.consecutive_failures++; + LOG(ERROR) << "Failed to send batch to Standby: " << standby_id + << ", consecutive failures: " << state.consecutive_failures; + + // Mark as TIMEOUT if too many failures + if (state.consecutive_failures >= kMaxConsecutiveFailures) { + state.state = StandbyHealthState::TIMEOUT; + LOG(WARNING) << "Standby " << standby_id + << " marked as TIMEOUT after " + << state.consecutive_failures << " failures"; + } else { + state.state = StandbyHealthState::SLOW; + } } } @@ -141,5 +167,74 @@ size_t ReplicationService::GetStandbyCount() const { return standbys_.size(); } +void ReplicationService::OnAck(const std::string& standby_id, uint64_t acked_seq_id) { + std::unique_lock lock(mutex_); + + auto it = standbys_.find(standby_id); + if (it == standbys_.end()) { + LOG(WARNING) << "Received ACK from unknown Standby: " << standby_id; + return; + } + + auto& state = it->second; + if (acked_seq_id > state.acked_seq_id) { + state.acked_seq_id = acked_seq_id; + state.last_ack_time = std::chrono::steady_clock::now(); + state.consecutive_failures = 0; // Reset failure count on successful ACK + + // Recover from SLOW or TIMEOUT state if we get an ACK + if (state.state == StandbyHealthState::SLOW || + state.state == StandbyHealthState::TIMEOUT) { + state.state = StandbyHealthState::HEALTHY; + LOG(INFO) << "Standby " << standby_id << " recovered, acked_seq_id=" + << acked_seq_id; + } + } +} + +void ReplicationService::CheckStandbyHealth() { + std::unique_lock lock(mutex_); + auto now = std::chrono::steady_clock::now(); + + for (auto& [standby_id, state] : standbys_) { + // Check connection status first + if (!state.stream || !state.stream->IsConnected()) { + if (state.state != StandbyHealthState::DISCONNECTED) { + state.state = StandbyHealthState::DISCONNECTED; + LOG(WARNING) << "Standby " << standby_id << " disconnected"; + } + continue; + } + + // Check ACK timeout + auto ack_age_ms = std::chrono::duration_cast( + now - state.last_ack_time).count(); + + if (ack_age_ms > kAckTimeoutMs) { + state.consecutive_failures++; + + if (state.consecutive_failures >= kMaxConsecutiveFailures) { + if (state.state != StandbyHealthState::TIMEOUT) { + state.state = StandbyHealthState::TIMEOUT; + LOG(WARNING) << "Standby " << standby_id + << " marked as TIMEOUT (ack_age=" << ack_age_ms + << "ms, failures=" << state.consecutive_failures << ")"; + } + } else { + if (state.state == StandbyHealthState::HEALTHY) { + state.state = StandbyHealthState::SLOW; + LOG(WARNING) << "Standby " << standby_id + << " is slow (ack_age=" << ack_age_ms << "ms)"; + } + } + } else { + // ACK received recently, reset failure count if healthy + if (state.state == StandbyHealthState::HEALTHY) { + state.consecutive_failures = 0; + } + } + } +} + } // namespace mooncake From de5c4fce3546a333e7f67fbebb7bda03bd99055d Mon Sep 17 00:00:00 2001 From: BernardLee Date: Wed, 24 Dec 2025 17:26:09 +0800 Subject: [PATCH 06/59] real Ack --- mooncake-store/include/replication_service.h | 18 ++++- mooncake-store/src/replication_service.cpp | 78 +++++++++++++++++++- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/mooncake-store/include/replication_service.h b/mooncake-store/include/replication_service.h index 2018fa06f9..ca1bd83895 100644 --- a/mooncake-store/include/replication_service.h +++ b/mooncake-store/include/replication_service.h @@ -102,6 +102,12 @@ class ReplicationService { */ std::map GetReplicationLag() const; + /** + * @brief Get the minimum sequence ID that has been ACKed by majority of Standbys + * @return Minimum majority-acked sequence ID, or 0 if no standbys + */ + uint64_t GetMajorityAckedSequenceId() const; + /** * @brief Get the number of connected Standbys */ @@ -120,6 +126,12 @@ class ReplicationService { */ void CheckStandbyHealth(); + /** + * @brief Truncate OpLog based on majority ACK + * This should be called periodically to free up memory + */ + void TruncateOpLog(); + private: /** * @brief Broadcast an OpLog entry to all connected Standbys @@ -147,11 +159,15 @@ class ReplicationService { struct StandbyState { std::shared_ptr stream; - uint64_t acked_seq_id{0}; // Last acknowledged sequence ID + uint64_t acked_seq_id{0}; // Last acknowledged sequence ID (only updated on real ACK) + uint64_t last_sent_seq_id{0}; // Last sent sequence ID std::chrono::steady_clock::time_point last_ack_time; std::chrono::steady_clock::time_point last_send_time; // Last send attempt time std::vector pending_batch; // Batched entries + // Track pending ACKs: map from seq_id to send_time + std::map pending_acks; + // Health monitoring StandbyHealthState state{StandbyHealthState::HEALTHY}; uint32_t consecutive_failures{0}; // Consecutive failure count diff --git a/mooncake-store/src/replication_service.cpp b/mooncake-store/src/replication_service.cpp index 6dfe7e4f0a..3ff64aa6b5 100644 --- a/mooncake-store/src/replication_service.cpp +++ b/mooncake-store/src/replication_service.cpp @@ -26,6 +26,7 @@ void ReplicationService::RegisterStandby(const std::string& standby_id, StandbyState state; state.stream = std::move(stream); state.acked_seq_id = 0; + state.last_sent_seq_id = 0; state.last_ack_time = std::chrono::steady_clock::now(); state.last_send_time = std::chrono::steady_clock::now(); state.state = StandbyHealthState::HEALTHY; @@ -99,17 +100,27 @@ void ReplicationService::SendBatch(const std::string& standby_id, // Update last send time state.last_send_time = std::chrono::steady_clock::now(); + auto send_time = std::chrono::steady_clock::now(); // For now, this is a placeholder. In the full implementation, // this would send via gRPC stream. bool success = state.stream->Send(entries); if (success) { // Reset failure count on successful send - // Note: ACK will be updated via OnAck() when we receive actual ACK state.consecutive_failures = 0; if (state.state == StandbyHealthState::SLOW) { state.state = StandbyHealthState::HEALTHY; } + + // Record pending ACKs (don't update acked_seq_id until we receive real ACK) + if (!entries.empty()) { + uint64_t last_seq_id = entries.back().sequence_id; + state.last_sent_seq_id = last_seq_id; + // Record all sequence IDs in this batch as pending + for (const auto& entry : entries) { + state.pending_acks[entry.sequence_id] = send_time; + } + } } else { state.consecutive_failures++; LOG(ERROR) << "Failed to send batch to Standby: " << standby_id @@ -182,6 +193,16 @@ void ReplicationService::OnAck(const std::string& standby_id, uint64_t acked_seq state.last_ack_time = std::chrono::steady_clock::now(); state.consecutive_failures = 0; // Reset failure count on successful ACK + // Clean up pending_acks that have been acknowledged + auto ack_it = state.pending_acks.begin(); + while (ack_it != state.pending_acks.end()) { + if (ack_it->first <= acked_seq_id) { + ack_it = state.pending_acks.erase(ack_it); + } else { + ++ack_it; + } + } + // Recover from SLOW or TIMEOUT state if we get an ACK if (state.state == StandbyHealthState::SLOW || state.state == StandbyHealthState::TIMEOUT) { @@ -210,6 +231,22 @@ void ReplicationService::CheckStandbyHealth() { auto ack_age_ms = std::chrono::duration_cast( now - state.last_ack_time).count(); + // Also check for stale pending ACKs + auto pending_it = state.pending_acks.begin(); + while (pending_it != state.pending_acks.end()) { + auto pending_age_ms = std::chrono::duration_cast( + now - pending_it->second).count(); + if (pending_age_ms > kAckTimeoutMs) { + // This pending ACK has timed out + LOG(WARNING) << "Pending ACK timeout for Standby " << standby_id + << ", seq_id=" << pending_it->first + << ", age=" << pending_age_ms << "ms"; + pending_it = state.pending_acks.erase(pending_it); + } else { + ++pending_it; + } + } + if (ack_age_ms > kAckTimeoutMs) { state.consecutive_failures++; @@ -236,5 +273,44 @@ void ReplicationService::CheckStandbyHealth() { } } +uint64_t ReplicationService::GetMajorityAckedSequenceId() const { + std::shared_lock lock(mutex_); + + if (standbys_.empty()) { + // No standbys, can truncate all + return oplog_manager_.GetLastSequenceId(); + } + + // Collect ACKed sequence IDs from healthy Standbys only + std::vector acked_ids; + for (const auto& [standby_id, state] : standbys_) { + // Only consider healthy or slow Standbys (not TIMEOUT or DISCONNECTED) + if (state.state == StandbyHealthState::HEALTHY || + state.state == StandbyHealthState::SLOW) { + acked_ids.push_back(state.acked_seq_id); + } + } + + if (acked_ids.empty()) { + // No healthy Standbys, cannot safely truncate + return 0; + } + + // Calculate majority (upward rounding) + size_t majority = (acked_ids.size() + 1) / 2; + + // Sort and return the majority-th smallest ACKed sequence ID + std::sort(acked_ids.begin(), acked_ids.end()); + return acked_ids[majority - 1]; +} + +void ReplicationService::TruncateOpLog() { + uint64_t majority_acked = GetMajorityAckedSequenceId(); + if (majority_acked > 0) { + oplog_manager_.TruncateBefore(majority_acked); + VLOG(1) << "Truncated OpLog before seq_id=" << majority_acked; + } +} + } // namespace mooncake From bb70f0715d98ba1b4879ddeb020a81f4d43d91ad Mon Sep 17 00:00:00 2001 From: BernardLee Date: Thu, 25 Dec 2025 10:13:03 +0800 Subject: [PATCH 07/59] add thread for check --- mooncake-store/include/replication_service.h | 27 +++++++++ mooncake-store/src/replication_service.cpp | 63 +++++++++++++++++++- mooncake-store/src/rpc_service.cpp | 10 +++- 3 files changed, 98 insertions(+), 2 deletions(-) diff --git a/mooncake-store/include/replication_service.h b/mooncake-store/include/replication_service.h index ca1bd83895..10b3c693e6 100644 --- a/mooncake-store/include/replication_service.h +++ b/mooncake-store/include/replication_service.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -132,6 +133,16 @@ class ReplicationService { */ void TruncateOpLog(); + /** + * @brief Start background threads for health checking and OpLog truncation + */ + void Start(); + + /** + * @brief Stop background threads + */ + void Stop(); + private: /** * @brief Broadcast an OpLog entry to all connected Standbys @@ -188,6 +199,22 @@ class ReplicationService { static constexpr uint32_t kSendTimeoutMs = 3000; // Send timeout (3 seconds) static constexpr uint32_t kMaxConsecutiveFailures = 3; // Max consecutive failures before marking as TIMEOUT static constexpr uint32_t kHealthCheckIntervalMs = 1000; // Health check interval (1 second) + static constexpr uint32_t kTruncateIntervalMs = 10000; // OpLog truncate interval (10 seconds) + + /** + * @brief Background thread function for health checking + */ + void HealthCheckThreadFunc(); + + /** + * @brief Background thread function for OpLog truncation + */ + void TruncateThreadFunc(); + + // Background threads + std::atomic running_{false}; + std::thread health_check_thread_; + std::thread truncate_thread_; }; } // namespace mooncake diff --git a/mooncake-store/src/replication_service.cpp b/mooncake-store/src/replication_service.cpp index 3ff64aa6b5..eadfaea909 100644 --- a/mooncake-store/src/replication_service.cpp +++ b/mooncake-store/src/replication_service.cpp @@ -4,6 +4,7 @@ #include #include +#include #include "master_service.h" #include "oplog_manager.h" @@ -12,13 +13,73 @@ namespace mooncake { ReplicationService::ReplicationService(OpLogManager& oplog_manager, MasterService& master_service) - : oplog_manager_(oplog_manager), master_service_(master_service) {} + : oplog_manager_(oplog_manager), master_service_(master_service) { + // Background threads will be started by Start() method +} ReplicationService::~ReplicationService() { + Stop(); std::unique_lock lock(mutex_); standbys_.clear(); } +void ReplicationService::Start() { + if (running_.load()) { + LOG(WARNING) << "ReplicationService is already running"; + return; + } + + running_.store(true); + health_check_thread_ = std::thread(&ReplicationService::HealthCheckThreadFunc, this); + truncate_thread_ = std::thread(&ReplicationService::TruncateThreadFunc, this); + LOG(INFO) << "ReplicationService background threads started"; +} + +void ReplicationService::Stop() { + if (!running_.load()) { + return; + } + + running_.store(false); + + if (health_check_thread_.joinable()) { + health_check_thread_.join(); + } + if (truncate_thread_.joinable()) { + truncate_thread_.join(); + } + + LOG(INFO) << "ReplicationService background threads stopped"; +} + +void ReplicationService::HealthCheckThreadFunc() { + LOG(INFO) << "ReplicationService health check thread started"; + + while (running_.load()) { + CheckStandbyHealth(); + + // Sleep for health check interval + std::this_thread::sleep_for( + std::chrono::milliseconds(kHealthCheckIntervalMs)); + } + + LOG(INFO) << "ReplicationService health check thread stopped"; +} + +void ReplicationService::TruncateThreadFunc() { + LOG(INFO) << "ReplicationService truncate thread started"; + + while (running_.load()) { + TruncateOpLog(); + + // Sleep for truncate interval + std::this_thread::sleep_for( + std::chrono::milliseconds(kTruncateIntervalMs)); + } + + LOG(INFO) << "ReplicationService truncate thread stopped"; +} + void ReplicationService::RegisterStandby(const std::string& standby_id, std::shared_ptr stream) { std::unique_lock lock(mutex_); diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index c9e1957a81..a9fbb2d3f1 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -38,7 +38,9 @@ WrappedMasterService::WrappedMasterService( master_service_.GetOpLogManager(), master_service_); // Set ReplicationService pointer in MasterService for OpLog notification master_service_.SetReplicationService(replication_service_.get()); - LOG(INFO) << "ReplicationService initialized for HA mode"; + // Start background threads for health checking and truncation + replication_service_->Start(); + LOG(INFO) << "ReplicationService initialized and started for HA mode"; } if (config.enable_metric_reporting) { @@ -59,6 +61,12 @@ WrappedMasterService::~WrappedMasterService() { if (metric_report_thread_.joinable()) { metric_report_thread_.join(); } + + // Stop ReplicationService if it was started + if (replication_service_) { + replication_service_->Stop(); + } + http_server_.stop(); } From 02950cf009e2d997bdb8b6501801cca4942b29fe Mon Sep 17 00:00:00 2001 From: BernardLee Date: Fri, 26 Dec 2025 15:35:17 +0800 Subject: [PATCH 08/59] based on etcd --- mooncake-store/include/etcd_oplog_store.h | 143 +++++++ mooncake-store/include/master_service.h | 15 +- mooncake-store/include/oplog_applier.h | 122 ++++++ mooncake-store/include/oplog_manager.h | 9 +- mooncake-store/include/oplog_watcher.h | 85 +++++ mooncake-store/include/replication_service.h | 221 ----------- mooncake-store/include/rpc_service.h | 11 +- mooncake-store/src/CMakeLists.txt | 2 +- mooncake-store/src/hot_standby_service.cpp | 5 +- mooncake-store/src/master_service.cpp | 37 +- mooncake-store/src/oplog_manager.cpp | 3 + mooncake-store/src/replication_service.cpp | 377 ------------------- mooncake-store/src/rpc_service.cpp | 18 +- 13 files changed, 394 insertions(+), 654 deletions(-) create mode 100644 mooncake-store/include/etcd_oplog_store.h create mode 100644 mooncake-store/include/oplog_applier.h create mode 100644 mooncake-store/include/oplog_watcher.h delete mode 100644 mooncake-store/include/replication_service.h delete mode 100644 mooncake-store/src/replication_service.cpp diff --git a/mooncake-store/include/etcd_oplog_store.h b/mooncake-store/include/etcd_oplog_store.h new file mode 100644 index 0000000000..cbf1f46964 --- /dev/null +++ b/mooncake-store/include/etcd_oplog_store.h @@ -0,0 +1,143 @@ +#pragma once + +#include +#include +#include + +#include "oplog_manager.h" + +namespace mooncake { + +/** + * @brief Store OpLog entries to etcd for reliable replication + * + * This class handles writing OpLog entries to etcd and provides methods + * for reading and managing OpLog entries in etcd. + */ +class EtcdOpLogStore { + public: + /** + * @brief Constructor + * @param etcd_endpoints Comma-separated etcd endpoints + * @param cluster_id Cluster identifier + */ + EtcdOpLogStore(const std::string& etcd_endpoints, + const std::string& cluster_id); + + ~EtcdOpLogStore(); + + /** + * @brief Write a single OpLog entry to etcd + * @param entry OpLog entry to write + * @return true on success, false on failure + */ + bool WriteOpLog(const OpLogEntry& entry); + + /** + * @brief Write multiple OpLog entries to etcd (batch operation) + * @param entries OpLog entries to write + * @return true on success, false on failure + */ + bool WriteOpLogBatch(const std::vector& entries); + + /** + * @brief Update the latest sequence ID in etcd + * @param sequence_id Latest sequence ID + * @return true on success, false on failure + */ + bool UpdateLatestSequenceId(uint64_t sequence_id); + + /** + * @brief Get the latest sequence ID from etcd + * @return Latest sequence ID, or 0 if not found + */ + uint64_t GetLatestSequenceId() const; + + /** + * @brief Record snapshot sequence ID + * @param snapshot_id Snapshot identifier + * @param sequence_id Sequence ID at snapshot time + * @return true on success, false on failure + */ + bool RecordSnapshotSequenceId(const std::string& snapshot_id, + uint64_t sequence_id); + + /** + * @brief Get snapshot sequence ID + * @param snapshot_id Snapshot identifier + * @return Sequence ID, or 0 if not found + */ + uint64_t GetSnapshotSequenceId(const std::string& snapshot_id) const; + + /** + * @brief Read OpLog entries from etcd since a given sequence ID + * @param start_seq_id Starting sequence ID (exclusive) + * @param limit Maximum number of entries to read + * @param entries Output vector of OpLog entries + * @return true on success, false on failure + */ + bool ReadOpLogSince(uint64_t start_seq_id, size_t limit, + std::vector& entries) const; + + /** + * @brief Read a single OpLog entry by sequence ID + * @param sequence_id Sequence ID + * @param entry Output OpLog entry + * @return true on success, false on failure + */ + bool ReadOpLogEntry(uint64_t sequence_id, OpLogEntry& entry) const; + + /** + * @brief Cleanup OpLog entries before a given sequence ID + * @param sequence_id Sequence ID (entries with seq_id < sequence_id will be deleted) + * @return true on success, false on failure + */ + bool CleanupOpLogBefore(uint64_t sequence_id); + + private: + /** + * @brief Build etcd key for OpLog entry + * @param sequence_id Sequence ID + * @return etcd key string + */ + std::string BuildOpLogKey(uint64_t sequence_id) const; + + /** + * @brief Build etcd key for latest sequence ID + * @return etcd key string + */ + std::string BuildLatestSequenceIdKey() const; + + /** + * @brief Build etcd key for snapshot sequence ID + * @param snapshot_id Snapshot identifier + * @return etcd key string + */ + std::string BuildSnapshotSequenceIdKey(const std::string& snapshot_id) const; + + /** + * @brief Serialize OpLog entry to JSON string + * @param entry OpLog entry + * @return JSON string + */ + std::string SerializeOpLogEntry(const OpLogEntry& entry) const; + + /** + * @brief Deserialize OpLog entry from JSON string + * @param data JSON string + * @param entry Output OpLog entry + * @return true on success, false on failure + */ + bool DeserializeOpLogEntry(const std::string& data, + OpLogEntry& entry) const; + + std::string etcd_endpoints_; + std::string cluster_id_; + std::string etcd_prefix_; // e.g., "mooncake-store/oplog" + + // etcd client will be added when implementing + // For now, we use EtcdHelper +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 412ec4001d..78da79b334 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -31,7 +31,7 @@ namespace mooncake { // Forward declarations class AllocationStrategy; class EvictionStrategy; -class ReplicationService; +// ReplicationService forward declaration removed - using etcd-based OpLog sync instead /* * @brief MasterService is the main class for the master server. @@ -274,16 +274,12 @@ class MasterService { tl::expected GetStorageConfig() const; /** - * @brief Get OpLogManager reference for external access (e.g., ReplicationService) + * @brief Get OpLogManager reference for external access * @return Reference to the OpLogManager instance */ OpLogManager& GetOpLogManager(); - /** - * @brief Set ReplicationService pointer for OpLog notification - * @param replication_service Pointer to ReplicationService (can be nullptr) - */ - void SetReplicationService(ReplicationService* replication_service); + // SetReplicationService removed - using etcd-based OpLog sync instead /** * @brief Mounts a file storage segment into the master. @@ -316,7 +312,7 @@ class MasterService { private: /** - * @brief Helper function to append OpLog entry and notify ReplicationService + * @brief Helper function to append OpLog entry * @param type Operation type * @param key Object key * @param payload Optional payload data @@ -659,8 +655,7 @@ class MasterService { // state-changing operations so that a standby master can replay them. OpLogManager oplog_manager_; - // ReplicationService pointer for OpLog notification (set by WrappedMasterService) - ReplicationService* replication_service_{nullptr}; + // ReplicationService removed - using etcd-based OpLog sync instead std::shared_ptr allocation_strategy_; diff --git a/mooncake-store/include/oplog_applier.h b/mooncake-store/include/oplog_applier.h new file mode 100644 index 0000000000..23fd2383a7 --- /dev/null +++ b/mooncake-store/include/oplog_applier.h @@ -0,0 +1,122 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" + +namespace mooncake { + +// Forward declaration +class MetadataStore; + +/** + * @brief Apply OpLog entries to Standby metadata store with ordering guarantee + * + * This class applies OpLog entries to the Standby metadata store, + * ensuring both global and per-key ordering. + */ +class OpLogApplier { + public: + /** + * @brief Constructor + * @param metadata_store Metadata store to apply changes to + */ + explicit OpLogApplier(MetadataStore* metadata_store); + + /** + * @brief Apply a single OpLog entry (with ordering checks) + * @param entry OpLog entry to apply + * @return true on success, false on failure or ordering violation + */ + bool ApplyOpLogEntry(const OpLogEntry& entry); + + /** + * @brief Apply multiple OpLog entries + * @param entries OpLog entries to apply + * @return Number of successfully applied entries + */ + size_t ApplyOpLogEntries(const std::vector& entries); + + /** + * @brief Get the current sequence ID for a key + * @param key Object key + * @return Current sequence ID, or 0 if key not found + */ + uint64_t GetKeySequenceId(const std::string& key) const; + + /** + * @brief Get the expected global sequence ID + * @return Expected global sequence ID + */ + uint64_t GetExpectedSequenceId() const; + + /** + * @brief Recover from a given sequence ID + * @param last_applied_sequence_id Last applied sequence ID + */ + void Recover(uint64_t last_applied_sequence_id); + + /** + * @brief Process pending entries (entries with non-continuous sequence IDs) + * @return Number of entries processed + */ + size_t ProcessPendingEntries(); + + private: + /** + * @brief Check if the entry's sequence order is valid + * @param entry OpLog entry + * @return true if order is valid, false otherwise + */ + bool CheckSequenceOrder(const OpLogEntry& entry); + + /** + * @brief Apply PUT_END operation + * @param entry OpLog entry + */ + void ApplyPutEnd(const OpLogEntry& entry); + + /** + * @brief Apply PUT_REVOKE operation + * @param entry OpLog entry + */ + void ApplyPutRevoke(const OpLogEntry& entry); + + /** + * @brief Apply REMOVE operation + * @param entry OpLog entry + */ + void ApplyRemove(const OpLogEntry& entry); + + /** + * @brief Request missing OpLog entry from etcd + * @param missing_seq_id Missing sequence ID + * @return true if entry was found and applied, false otherwise + */ + bool RequestMissingOpLog(uint64_t missing_seq_id); + + /** + * @brief Schedule wait for missing entries + * @param missing_seq_id Missing sequence ID + */ + void ScheduleWaitForMissingEntries(uint64_t missing_seq_id); + + MetadataStore* metadata_store_; + + // Track per-key sequence ID for ordering guarantee + mutable std::mutex key_sequence_mutex_; + std::unordered_map key_sequence_map_; + + // Track pending entries (entries with non-continuous sequence IDs) + mutable std::mutex pending_mutex_; + std::map pending_entries_; + uint64_t expected_sequence_id_{1}; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index 1d9e26ded5..f971b1e13f 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace mooncake { @@ -20,13 +21,14 @@ enum class OpType : uint8_t { // A single operation log entry. struct OpLogEntry { - uint64_t sequence_id{0}; // Monotonically increasing sequence + uint64_t sequence_id{0}; // Monotonically increasing global sequence uint64_t timestamp_ms{0}; // Logical timestamp in milliseconds OpType op_type{OpType::PUT_END}; std::string object_key; // Target object key std::string payload; // Serialized extra data (optional) uint32_t checksum{0}; // Checksum of payload (implementation-defined) uint32_t prefix_hash{0}; // Hash of key prefix (for future verification) + uint64_t key_sequence_id{0}; // Per-key sequence ID (for ordering guarantee) }; /** @@ -34,7 +36,7 @@ struct OpLogEntry { * * This class is intentionally simple: it keeps a bounded deque of OpLogEntry * and provides append / get-since primitives. It can later be extended to - * notify ReplicationService or to spill to disk if needed. + * or to spill to disk if needed. In the new etcd-based design, OpLog will be written to etcd. */ class OpLogManager { public: @@ -66,6 +68,9 @@ class OpLogManager { std::deque buffer_; uint64_t first_seq_id_{1}; // sequence_id of buffer_.front() uint64_t last_seq_id_{0}; // last assigned sequence_id + + // Track per-key sequence ID for ordering guarantee + std::unordered_map key_sequence_map_; // Simple bounds to avoid unbounded memory growth. static constexpr size_t kMaxBufferEntries_ = 100000; diff --git a/mooncake-store/include/oplog_watcher.h b/mooncake-store/include/oplog_watcher.h new file mode 100644 index 0000000000..9357464d81 --- /dev/null +++ b/mooncake-store/include/oplog_watcher.h @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" + +namespace mooncake { + +// Forward declaration +class OpLogApplier; + +/** + * @brief Watch etcd for OpLog changes and apply them to Standby + * + * This class watches etcd for new OpLog entries and forwards them + * to OpLogApplier for processing. + */ +class OpLogWatcher { + public: + /** + * @brief Constructor + * @param etcd_endpoints Comma-separated etcd endpoints + * @param cluster_id Cluster identifier + * @param applier OpLog applier to process entries + */ + OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier); + + ~OpLogWatcher(); + + /** + * @brief Start watching etcd for OpLog changes + */ + void Start(); + + /** + * @brief Stop watching + */ + void Stop(); + + /** + * @brief Read OpLog entries from etcd since a given sequence ID + * @param start_seq_id Starting sequence ID (exclusive) + * @param entries Output vector of OpLog entries + * @return true on success, false on failure + */ + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries); + + /** + * @brief Get the last processed sequence ID + * @return Last processed sequence ID + */ + uint64_t GetLastProcessedSequenceId() const; + + private: + /** + * @brief Watch etcd OpLog changes (runs in background thread) + */ + void WatchOpLog(); + + /** + * @brief Process a Watch event + * @param key etcd key + * @param value etcd value + * @param revision etcd revision + */ + void HandleWatchEvent(const std::string& key, const std::string& value, + int64_t revision); + + std::string etcd_endpoints_; + std::string cluster_id_; + OpLogApplier* applier_; + std::atomic running_{false}; + std::thread watch_thread_; + std::atomic last_processed_sequence_id_{0}; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/replication_service.h b/mooncake-store/include/replication_service.h deleted file mode 100644 index 10b3c693e6..0000000000 --- a/mooncake-store/include/replication_service.h +++ /dev/null @@ -1,221 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "oplog_manager.h" -#include "types.h" - -namespace mooncake { - -// Forward declarations -class MasterService; -class OpLogManager; - -/** - * @brief Replication stream interface (placeholder for future gRPC implementation) - * - * This is a minimal interface that will be replaced with actual gRPC streaming - * in the future. For now, it serves as a placeholder to establish the - * architecture. - */ -class ReplicationStream { - public: - virtual ~ReplicationStream() = default; - virtual bool Send(const std::vector& entries) = 0; - virtual bool IsConnected() const = 0; -}; - -/** - * @brief Verification request/response structures - */ -struct VerificationRequest { - std::string standby_id; - std::vector> samples; // (key, checksum) - uint32_t prefix_hash; -}; - -struct VerificationResponse { - std::vector mismatched_keys; // Keys with checksum mismatch - bool is_consistent; -}; - -/** - * @brief ReplicationService manages OpLog replication from Primary to Standbys - * - * This service runs on the Primary Master and is responsible for: - * - Broadcasting OpLog entries to all connected Standby nodes - * - Tracking replication lag for each Standby - * - Handling verification requests from Standbys - * - * For now, this is a skeleton implementation without actual network - * communication. The gRPC integration will be added later. - */ -class ReplicationService { - public: - explicit ReplicationService(OpLogManager& oplog_manager, - MasterService& master_service); - - ~ReplicationService(); - - /** - * @brief Register a new Standby connection - * @param standby_id Unique identifier for the Standby node - * @param stream Replication stream for sending OpLog entries - */ - void RegisterStandby(const std::string& standby_id, - std::shared_ptr stream); - - /** - * @brief Unregister a Standby (when it disconnects) - * @param standby_id Unique identifier for the Standby node - */ - void UnregisterStandby(const std::string& standby_id); - - /** - * @brief Called by OpLogManager when a new entry is appended - * @param entry The newly appended OpLog entry - * - * This method should be called by OpLogManager (via callback) or - * directly from MasterService after appending to OpLog. - */ - void OnNewOpLog(const OpLogEntry& entry); - - /** - * @brief Handle verification request from a Standby - * @param request Verification request containing checksums - * @return Verification response with mismatched keys - */ - VerificationResponse HandleVerification(const VerificationRequest& request); - - /** - * @brief Get replication lag for each Standby - * @return Map of standby_id -> lag in sequence IDs - */ - std::map GetReplicationLag() const; - - /** - * @brief Get the minimum sequence ID that has been ACKed by majority of Standbys - * @return Minimum majority-acked sequence ID, or 0 if no standbys - */ - uint64_t GetMajorityAckedSequenceId() const; - - /** - * @brief Get the number of connected Standbys - */ - size_t GetStandbyCount() const; - - /** - * @brief Handle ACK from Standby - * @param standby_id Standby identifier - * @param acked_seq_id Acknowledged sequence ID - */ - void OnAck(const std::string& standby_id, uint64_t acked_seq_id); - - /** - * @brief Check health status of all Standbys and update their states - * This should be called periodically (e.g., by a background thread) - */ - void CheckStandbyHealth(); - - /** - * @brief Truncate OpLog based on majority ACK - * This should be called periodically to free up memory - */ - void TruncateOpLog(); - - /** - * @brief Start background threads for health checking and OpLog truncation - */ - void Start(); - - /** - * @brief Stop background threads - */ - void Stop(); - - private: - /** - * @brief Broadcast an OpLog entry to all connected Standbys - * @param entry The OpLog entry to broadcast - */ - void BroadcastEntry(const OpLogEntry& entry); - - /** - * @brief Send a batch of OpLog entries to a specific Standby - * @param standby_id Target Standby identifier - * @param entries Batch of OpLog entries to send - */ - void SendBatch(const std::string& standby_id, - const std::vector& entries); - - /** - * @brief State for each connected Standby - */ - enum class StandbyHealthState { - HEALTHY, // Normal state, responding to requests - SLOW, // Responding slowly, but still processing - TIMEOUT, // Timeout detected, possibly failed - DISCONNECTED // Connection lost - }; - - struct StandbyState { - std::shared_ptr stream; - uint64_t acked_seq_id{0}; // Last acknowledged sequence ID (only updated on real ACK) - uint64_t last_sent_seq_id{0}; // Last sent sequence ID - std::chrono::steady_clock::time_point last_ack_time; - std::chrono::steady_clock::time_point last_send_time; // Last send attempt time - std::vector pending_batch; // Batched entries - - // Track pending ACKs: map from seq_id to send_time - std::map pending_acks; - - // Health monitoring - StandbyHealthState state{StandbyHealthState::HEALTHY}; - uint32_t consecutive_failures{0}; // Consecutive failure count - }; - - OpLogManager& oplog_manager_; - MasterService& master_service_; - - mutable std::shared_mutex mutex_; - std::unordered_map standbys_; - - // Batch configuration - static constexpr size_t kBatchSize = 100; - static constexpr uint32_t kBatchTimeoutMs = 10; - - // Health check configuration - static constexpr uint32_t kAckTimeoutMs = 5000; // ACK timeout (5 seconds) - static constexpr uint32_t kSendTimeoutMs = 3000; // Send timeout (3 seconds) - static constexpr uint32_t kMaxConsecutiveFailures = 3; // Max consecutive failures before marking as TIMEOUT - static constexpr uint32_t kHealthCheckIntervalMs = 1000; // Health check interval (1 second) - static constexpr uint32_t kTruncateIntervalMs = 10000; // OpLog truncate interval (10 seconds) - - /** - * @brief Background thread function for health checking - */ - void HealthCheckThreadFunc(); - - /** - * @brief Background thread function for OpLog truncation - */ - void TruncateThreadFunc(); - - // Background threads - std::atomic running_{false}; - std::thread health_check_thread_; - std::thread truncate_thread_; -}; - -} // namespace mooncake - diff --git a/mooncake-store/include/rpc_service.h b/mooncake-store/include/rpc_service.h index 1013398d8c..7e0aef3a7d 100644 --- a/mooncake-store/include/rpc_service.h +++ b/mooncake-store/include/rpc_service.h @@ -15,7 +15,7 @@ // Forward declaration namespace mooncake { -class ReplicationService; +// ReplicationService forward declaration removed - using etcd-based OpLog sync instead } namespace mooncake { @@ -114,11 +114,7 @@ class WrappedMasterService { const UUID& client_id, const std::vector& keys, const std::vector& metadatas); - /** - * @brief Get ReplicationService pointer for RPC layer access - * @return Pointer to ReplicationService, or nullptr if not initialized - */ - ReplicationService* GetReplicationService(); + // GetReplicationService removed - using etcd-based OpLog sync instead private: MasterService master_service_; @@ -126,8 +122,7 @@ class WrappedMasterService { coro_http::coro_http_server http_server_; std::atomic metric_report_running_; - // ReplicationService for hot-standby replication (only initialized in HA mode) - std::unique_ptr replication_service_; + // ReplicationService removed - using etcd-based OpLog sync instead }; void RegisterRpcService(coro_rpc::coro_rpc_server& server, diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index a20b889981..603e1d52be 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -27,7 +27,7 @@ set(MOONCAKE_STORE_SOURCES http_metadata_server.cpp file_storage.cpp oplog_manager.cpp - replication_service.cpp + // replication_service.cpp removed - using etcd-based OpLog sync instead ) set(EXTRA_LIBS "") diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index 6e6e973173..ab07d99205 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -200,7 +200,10 @@ void HotStandbyService::ApplyOpLogEntry(const OpLogEntry& entry) { } break; case OpType::LEASE_RENEW: - // Update lease timeout (no change to entry count) + // LEASE_RENEW is no longer used. Standby does not perform eviction, + // so it doesn't need to track lease renewals. DELETE events from + // Primary will handle object removal. + // This case is kept for backward compatibility with old OpLog entries. break; default: LOG(WARNING) << "Unknown OpType: " diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 2ddd594795..c941e64bc1 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -10,7 +10,7 @@ #include "master_metric_manager.h" #include "segment.h" #include "types.h" -#include "replication_service.h" +// replication_service.h removed - using etcd-based OpLog sync instead namespace mooncake { @@ -75,19 +75,13 @@ MasterService::MasterService(const MasterServiceConfig& config) } } -// Helper function to append OpLog entry and notify ReplicationService +// Helper function to append OpLog entry +// Note: In the new etcd-based design, OpLog will be written to etcd by EtcdOpLogStore +// This method only appends to OpLogManager's buffer for now void MasterService::AppendOpLogAndNotify(OpType type, const std::string& key, const std::string& payload) { - uint64_t seq_id = oplog_manager_.Append(type, key, payload); - - // Notify ReplicationService if it's set - if (replication_service_ != nullptr) { - // Get the entry we just appended (GetEntriesSince returns entries with seq_id > since_seq_id) - auto entries = oplog_manager_.GetEntriesSince(seq_id - 1, 1); - if (!entries.empty() && entries[0].sequence_id == seq_id) { - replication_service_->OnNewOpLog(entries[0]); - } - } + oplog_manager_.Append(type, key, payload); + // TODO: In Phase 1, integrate with EtcdOpLogStore to write to etcd } MasterService::~MasterService() { @@ -247,8 +241,9 @@ auto MasterService::ExistKey(const std::string& key) // client. metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); - // Record lease renewal for standby synchronization. - AppendOpLogAndNotify(OpType::LEASE_RENEW, key); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. return true; } } @@ -519,8 +514,9 @@ auto MasterService::GetReplicaListByRegex(const std::string& regex_pattern) results.emplace(key, std::move(replica_list)); metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); - // Record lease renewal for standby synchronization. - AppendOpLogAndNotify(OpType::LEASE_RENEW, key); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. } } } @@ -562,8 +558,9 @@ auto MasterService::GetReplicaList(std::string_view key) // Grant a lease to the object so it will not be removed // when the client is reading it. metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); - // Record lease renewal for standby synchronization. - AppendOpLogAndNotify(OpType::LEASE_RENEW, std::string(key)); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. return GetReplicaListResponse(std::move(replica_list), default_kv_lease_ttl_); @@ -1599,8 +1596,6 @@ OpLogManager& MasterService::GetOpLogManager() { return oplog_manager_; } -void MasterService::SetReplicationService(ReplicationService* replication_service) { - replication_service_ = replication_service; -} +// SetReplicationService removed - using etcd-based OpLog sync instead } // namespace mooncake \ No newline at end of file diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index 54727e51d5..7a12ef8229 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -21,6 +21,9 @@ uint64_t OpLogManager::Append(OpType type, const std::string& key, std::unique_lock lock(mutex_); entry.sequence_id = ++last_seq_id_; + + // Track per-key sequence ID for ordering guarantee + entry.key_sequence_id = ++key_sequence_map_[key]; if (buffer_.size() >= kMaxBufferEntries_) { buffer_.pop_front(); diff --git a/mooncake-store/src/replication_service.cpp b/mooncake-store/src/replication_service.cpp deleted file mode 100644 index eadfaea909..0000000000 --- a/mooncake-store/src/replication_service.cpp +++ /dev/null @@ -1,377 +0,0 @@ -#include "replication_service.h" - -#include - -#include -#include -#include - -#include "master_service.h" -#include "oplog_manager.h" - -namespace mooncake { - -ReplicationService::ReplicationService(OpLogManager& oplog_manager, - MasterService& master_service) - : oplog_manager_(oplog_manager), master_service_(master_service) { - // Background threads will be started by Start() method -} - -ReplicationService::~ReplicationService() { - Stop(); - std::unique_lock lock(mutex_); - standbys_.clear(); -} - -void ReplicationService::Start() { - if (running_.load()) { - LOG(WARNING) << "ReplicationService is already running"; - return; - } - - running_.store(true); - health_check_thread_ = std::thread(&ReplicationService::HealthCheckThreadFunc, this); - truncate_thread_ = std::thread(&ReplicationService::TruncateThreadFunc, this); - LOG(INFO) << "ReplicationService background threads started"; -} - -void ReplicationService::Stop() { - if (!running_.load()) { - return; - } - - running_.store(false); - - if (health_check_thread_.joinable()) { - health_check_thread_.join(); - } - if (truncate_thread_.joinable()) { - truncate_thread_.join(); - } - - LOG(INFO) << "ReplicationService background threads stopped"; -} - -void ReplicationService::HealthCheckThreadFunc() { - LOG(INFO) << "ReplicationService health check thread started"; - - while (running_.load()) { - CheckStandbyHealth(); - - // Sleep for health check interval - std::this_thread::sleep_for( - std::chrono::milliseconds(kHealthCheckIntervalMs)); - } - - LOG(INFO) << "ReplicationService health check thread stopped"; -} - -void ReplicationService::TruncateThreadFunc() { - LOG(INFO) << "ReplicationService truncate thread started"; - - while (running_.load()) { - TruncateOpLog(); - - // Sleep for truncate interval - std::this_thread::sleep_for( - std::chrono::milliseconds(kTruncateIntervalMs)); - } - - LOG(INFO) << "ReplicationService truncate thread stopped"; -} - -void ReplicationService::RegisterStandby(const std::string& standby_id, - std::shared_ptr stream) { - std::unique_lock lock(mutex_); - - StandbyState state; - state.stream = std::move(stream); - state.acked_seq_id = 0; - state.last_sent_seq_id = 0; - state.last_ack_time = std::chrono::steady_clock::now(); - state.last_send_time = std::chrono::steady_clock::now(); - state.state = StandbyHealthState::HEALTHY; - state.consecutive_failures = 0; - - standbys_[standby_id] = std::move(state); - - LOG(INFO) << "Registered Standby: " << standby_id - << ", total standbys: " << standbys_.size(); -} - -void ReplicationService::UnregisterStandby(const std::string& standby_id) { - std::unique_lock lock(mutex_); - - auto it = standbys_.find(standby_id); - if (it != standbys_.end()) { - standbys_.erase(it); - LOG(INFO) << "Unregistered Standby: " << standby_id - << ", remaining standbys: " << standbys_.size(); - } -} - -void ReplicationService::OnNewOpLog(const OpLogEntry& entry) { - std::shared_lock lock(mutex_); - - if (standbys_.empty()) { - // No standbys connected, nothing to do - return; - } - - // Broadcast to all standbys - BroadcastEntry(entry); -} - -void ReplicationService::BroadcastEntry(const OpLogEntry& entry) { - // Skip Standbys that are in TIMEOUT or DISCONNECTED state - for (auto& [standby_id, state] : standbys_) { - // Skip unhealthy Standbys - if (state.state == StandbyHealthState::TIMEOUT || - state.state == StandbyHealthState::DISCONNECTED) { - continue; - } - - state.pending_batch.push_back(entry); - - // If batch is full, send it immediately - if (state.pending_batch.size() >= kBatchSize) { - SendBatch(standby_id, state.pending_batch); - state.pending_batch.clear(); - } - } -} - -void ReplicationService::SendBatch(const std::string& standby_id, - const std::vector& entries) { - auto it = standbys_.find(standby_id); - if (it == standbys_.end()) { - LOG(WARNING) << "Attempted to send batch to unknown Standby: " - << standby_id; - return; - } - - auto& state = it->second; - - // Check connection status - if (!state.stream || !state.stream->IsConnected()) { - state.state = StandbyHealthState::DISCONNECTED; - LOG(WARNING) << "Standby stream not connected: " << standby_id; - return; - } - - // Update last send time - state.last_send_time = std::chrono::steady_clock::now(); - auto send_time = std::chrono::steady_clock::now(); - - // For now, this is a placeholder. In the full implementation, - // this would send via gRPC stream. - bool success = state.stream->Send(entries); - if (success) { - // Reset failure count on successful send - state.consecutive_failures = 0; - if (state.state == StandbyHealthState::SLOW) { - state.state = StandbyHealthState::HEALTHY; - } - - // Record pending ACKs (don't update acked_seq_id until we receive real ACK) - if (!entries.empty()) { - uint64_t last_seq_id = entries.back().sequence_id; - state.last_sent_seq_id = last_seq_id; - // Record all sequence IDs in this batch as pending - for (const auto& entry : entries) { - state.pending_acks[entry.sequence_id] = send_time; - } - } - } else { - state.consecutive_failures++; - LOG(ERROR) << "Failed to send batch to Standby: " << standby_id - << ", consecutive failures: " << state.consecutive_failures; - - // Mark as TIMEOUT if too many failures - if (state.consecutive_failures >= kMaxConsecutiveFailures) { - state.state = StandbyHealthState::TIMEOUT; - LOG(WARNING) << "Standby " << standby_id - << " marked as TIMEOUT after " - << state.consecutive_failures << " failures"; - } else { - state.state = StandbyHealthState::SLOW; - } - } -} - -VerificationResponse ReplicationService::HandleVerification( - const VerificationRequest& request) { - VerificationResponse response; - response.is_consistent = true; - - // TODO: Implement actual verification logic by comparing checksums - // with MasterService metadata. For now, this is a placeholder. - - LOG(INFO) << "Verification request from Standby: " << request.standby_id - << ", samples: " << request.samples.size(); - - // Placeholder: assume consistent for now - response.is_consistent = true; - response.mismatched_keys.clear(); - - return response; -} - -std::map ReplicationService::GetReplicationLag() const { - std::shared_lock lock(mutex_); - std::map lag_map; - - uint64_t primary_seq_id = oplog_manager_.GetLastSequenceId(); - - for (const auto& [standby_id, state] : standbys_) { - uint64_t lag = 0; - if (primary_seq_id > state.acked_seq_id) { - lag = primary_seq_id - state.acked_seq_id; - } - lag_map[standby_id] = lag; - } - - return lag_map; -} - -size_t ReplicationService::GetStandbyCount() const { - std::shared_lock lock(mutex_); - return standbys_.size(); -} - -void ReplicationService::OnAck(const std::string& standby_id, uint64_t acked_seq_id) { - std::unique_lock lock(mutex_); - - auto it = standbys_.find(standby_id); - if (it == standbys_.end()) { - LOG(WARNING) << "Received ACK from unknown Standby: " << standby_id; - return; - } - - auto& state = it->second; - if (acked_seq_id > state.acked_seq_id) { - state.acked_seq_id = acked_seq_id; - state.last_ack_time = std::chrono::steady_clock::now(); - state.consecutive_failures = 0; // Reset failure count on successful ACK - - // Clean up pending_acks that have been acknowledged - auto ack_it = state.pending_acks.begin(); - while (ack_it != state.pending_acks.end()) { - if (ack_it->first <= acked_seq_id) { - ack_it = state.pending_acks.erase(ack_it); - } else { - ++ack_it; - } - } - - // Recover from SLOW or TIMEOUT state if we get an ACK - if (state.state == StandbyHealthState::SLOW || - state.state == StandbyHealthState::TIMEOUT) { - state.state = StandbyHealthState::HEALTHY; - LOG(INFO) << "Standby " << standby_id << " recovered, acked_seq_id=" - << acked_seq_id; - } - } -} - -void ReplicationService::CheckStandbyHealth() { - std::unique_lock lock(mutex_); - auto now = std::chrono::steady_clock::now(); - - for (auto& [standby_id, state] : standbys_) { - // Check connection status first - if (!state.stream || !state.stream->IsConnected()) { - if (state.state != StandbyHealthState::DISCONNECTED) { - state.state = StandbyHealthState::DISCONNECTED; - LOG(WARNING) << "Standby " << standby_id << " disconnected"; - } - continue; - } - - // Check ACK timeout - auto ack_age_ms = std::chrono::duration_cast( - now - state.last_ack_time).count(); - - // Also check for stale pending ACKs - auto pending_it = state.pending_acks.begin(); - while (pending_it != state.pending_acks.end()) { - auto pending_age_ms = std::chrono::duration_cast( - now - pending_it->second).count(); - if (pending_age_ms > kAckTimeoutMs) { - // This pending ACK has timed out - LOG(WARNING) << "Pending ACK timeout for Standby " << standby_id - << ", seq_id=" << pending_it->first - << ", age=" << pending_age_ms << "ms"; - pending_it = state.pending_acks.erase(pending_it); - } else { - ++pending_it; - } - } - - if (ack_age_ms > kAckTimeoutMs) { - state.consecutive_failures++; - - if (state.consecutive_failures >= kMaxConsecutiveFailures) { - if (state.state != StandbyHealthState::TIMEOUT) { - state.state = StandbyHealthState::TIMEOUT; - LOG(WARNING) << "Standby " << standby_id - << " marked as TIMEOUT (ack_age=" << ack_age_ms - << "ms, failures=" << state.consecutive_failures << ")"; - } - } else { - if (state.state == StandbyHealthState::HEALTHY) { - state.state = StandbyHealthState::SLOW; - LOG(WARNING) << "Standby " << standby_id - << " is slow (ack_age=" << ack_age_ms << "ms)"; - } - } - } else { - // ACK received recently, reset failure count if healthy - if (state.state == StandbyHealthState::HEALTHY) { - state.consecutive_failures = 0; - } - } - } -} - -uint64_t ReplicationService::GetMajorityAckedSequenceId() const { - std::shared_lock lock(mutex_); - - if (standbys_.empty()) { - // No standbys, can truncate all - return oplog_manager_.GetLastSequenceId(); - } - - // Collect ACKed sequence IDs from healthy Standbys only - std::vector acked_ids; - for (const auto& [standby_id, state] : standbys_) { - // Only consider healthy or slow Standbys (not TIMEOUT or DISCONNECTED) - if (state.state == StandbyHealthState::HEALTHY || - state.state == StandbyHealthState::SLOW) { - acked_ids.push_back(state.acked_seq_id); - } - } - - if (acked_ids.empty()) { - // No healthy Standbys, cannot safely truncate - return 0; - } - - // Calculate majority (upward rounding) - size_t majority = (acked_ids.size() + 1) / 2; - - // Sort and return the majority-th smallest ACKed sequence ID - std::sort(acked_ids.begin(), acked_ids.end()); - return acked_ids[majority - 1]; -} - -void ReplicationService::TruncateOpLog() { - uint64_t majority_acked = GetMajorityAckedSequenceId(); - if (majority_acked > 0) { - oplog_manager_.TruncateBefore(majority_acked); - VLOG(1) << "Truncated OpLog before seq_id=" << majority_acked; - } -} - -} // namespace mooncake - diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index a9fbb2d3f1..6aa9cb86bd 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -19,7 +19,7 @@ #include "types.h" #include "utils/scoped_vlog_timer.h" #include "version.h" -#include "replication_service.h" +// replication_service.h removed - using etcd-based OpLog sync instead namespace mooncake { @@ -32,15 +32,10 @@ WrappedMasterService::WrappedMasterService( metric_report_running_(config.enable_metric_reporting) { init_http_server(); - // Initialize ReplicationService if HA mode is enabled + // ReplicationService removed - using etcd-based OpLog sync instead + // TODO: In Phase 1, initialize EtcdOpLogStore and integrate with OpLogManager if (config.enable_ha) { - replication_service_ = std::make_unique( - master_service_.GetOpLogManager(), master_service_); - // Set ReplicationService pointer in MasterService for OpLog notification - master_service_.SetReplicationService(replication_service_.get()); - // Start background threads for health checking and truncation - replication_service_->Start(); - LOG(INFO) << "ReplicationService initialized and started for HA mode"; + LOG(INFO) << "HA mode enabled - etcd-based OpLog sync will be implemented in Phase 1"; } if (config.enable_metric_reporting) { @@ -62,10 +57,7 @@ WrappedMasterService::~WrappedMasterService() { metric_report_thread_.join(); } - // Stop ReplicationService if it was started - if (replication_service_) { - replication_service_->Stop(); - } + // ReplicationService removed - using etcd-based OpLog sync instead http_server_.stop(); } From 2cafab1b9b3a9a37d33269b73d595738ccae483d Mon Sep 17 00:00:00 2001 From: BernardLee Date: Fri, 26 Dec 2025 15:35:59 +0800 Subject: [PATCH 09/59] based on etcd 2 --- doc/en/diagrams/oplog-data-flow.puml | 58 ++ doc/en/diagrams/oplog-failover-sequence.puml | 66 ++ .../oplog-hot-standby-architecture.puml | 65 ++ doc/en/rfc-oplog-hot-standby-complete.md | 318 ++++++++ doc/zh/diagrams/mooncake-transfer-flow.puml | 80 ++ doc/zh/diagrams/oplog-data-flow.puml | 58 ++ doc/zh/diagrams/oplog-failover-sequence.puml | 66 ++ .../oplog-hot-standby-architecture.puml | 65 ++ ...log-hot-standby-complete-architecture.puml | 123 +++ doc/zh/rfc-batched-delete-events-via-etcd.md | 455 +++++++++++ doc/zh/rfc-batched-delete-timing-issues.md | 419 ++++++++++ doc/zh/rfc-delete-via-etcd-solution.md | 427 ++++++++++ .../rfc-dragonflydb-as-consistency-store.md | 313 ++++++++ doc/zh/rfc-oplog-cleanup-start-sequence-id.md | 507 ++++++++++++ doc/zh/rfc-oplog-hot-standby-complete.md | 364 +++++++++ doc/zh/rfc-oplog-hot-standby-promotion.md | 41 + doc/zh/rfc-oplog-implementation-plan.md | 712 +++++++++++++++++ doc/zh/rfc-oplog-key-sequence-map-cleanup.md | 253 ++++++ ...g-rollback-replay-on-sequence-violation.md | 653 ++++++++++++++++ doc/zh/rfc-oplog-via-etcd-complete-design.md | 738 ++++++++++++++++++ doc/zh/rfc-standby-no-response-handling.md | 355 +++++++++ ...-standby-promotion-lease-initialization.md | 439 +++++++++++ doc/zh/rfc-standby-service-integration.md | 673 ++++++++++++++++ 23 files changed, 7248 insertions(+) create mode 100644 doc/en/diagrams/oplog-data-flow.puml create mode 100644 doc/en/diagrams/oplog-failover-sequence.puml create mode 100644 doc/en/diagrams/oplog-hot-standby-architecture.puml create mode 100644 doc/en/rfc-oplog-hot-standby-complete.md create mode 100644 doc/zh/diagrams/mooncake-transfer-flow.puml create mode 100644 doc/zh/diagrams/oplog-data-flow.puml create mode 100644 doc/zh/diagrams/oplog-failover-sequence.puml create mode 100644 doc/zh/diagrams/oplog-hot-standby-architecture.puml create mode 100644 doc/zh/diagrams/oplog-hot-standby-complete-architecture.puml create mode 100644 doc/zh/rfc-batched-delete-events-via-etcd.md create mode 100644 doc/zh/rfc-batched-delete-timing-issues.md create mode 100644 doc/zh/rfc-delete-via-etcd-solution.md create mode 100644 doc/zh/rfc-dragonflydb-as-consistency-store.md create mode 100644 doc/zh/rfc-oplog-cleanup-start-sequence-id.md create mode 100644 doc/zh/rfc-oplog-hot-standby-complete.md create mode 100644 doc/zh/rfc-oplog-hot-standby-promotion.md create mode 100644 doc/zh/rfc-oplog-implementation-plan.md create mode 100644 doc/zh/rfc-oplog-key-sequence-map-cleanup.md create mode 100644 doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md create mode 100644 doc/zh/rfc-oplog-via-etcd-complete-design.md create mode 100644 doc/zh/rfc-standby-no-response-handling.md create mode 100644 doc/zh/rfc-standby-promotion-lease-initialization.md create mode 100644 doc/zh/rfc-standby-service-integration.md diff --git a/doc/en/diagrams/oplog-data-flow.puml b/doc/en/diagrams/oplog-data-flow.puml new file mode 100644 index 0000000000..32682eda9e --- /dev/null +++ b/doc/en/diagrams/oplog-data-flow.puml @@ -0,0 +1,58 @@ +@startuml oplog-data-flow +!theme plain +skinparam backgroundColor #FFFFFF +skinparam sequenceMessageAlign center + +actor Client +participant "Primary Master" as Primary +participant "OpLogManager" as OplogMgr +participant "EtcdOpLogStore" as EtcdStore +database etcd +participant "OpLogWatcher" as Watcher +participant "OpLogApplier" as Applier +participant "Standby Master" as Standby + +== PUT_END Operation Flow == + +Client -> Primary: PutEnd(key, ...) +activate Primary +Primary -> OplogMgr: Append(PUT_END, key) +activate OplogMgr +OplogMgr -> OplogMgr: Generate sequence_id\nGenerate key_sequence_id +OplogMgr -> EtcdStore: WriteOpLog(entry) +activate EtcdStore +EtcdStore -> etcd: PUT /oplog/{seq} +activate etcd +etcd --> EtcdStore: Success +deactivate etcd +EtcdStore --> OplogMgr: Success +deactivate EtcdStore +OplogMgr --> Primary: sequence_id +deactivate OplogMgr +Primary --> Client: Success +deactivate Primary + +== Standby Synchronization Flow == + +etcd -> Watcher: Watch Event (New OpLog) +activate Watcher +Watcher -> Applier: ApplyOpLogEntry(entry) +activate Applier +Applier -> Applier: CheckSequenceOrder() +alt Order Correct + Applier -> Standby: UpdateMetadata(key, ...) + activate Standby + Standby --> Applier: Success + deactivate Standby +else Order Violation + Applier -> Applier: RollbackAndReplay() + Applier -> etcd: ReadOpLogForKey() + etcd --> Applier: OpLog Entries + Applier -> Applier: Replay OpLog +end +Applier --> Watcher: Success +deactivate Applier +deactivate Watcher + +@enduml + diff --git a/doc/en/diagrams/oplog-failover-sequence.puml b/doc/en/diagrams/oplog-failover-sequence.puml new file mode 100644 index 0000000000..f6da4cc1f8 --- /dev/null +++ b/doc/en/diagrams/oplog-failover-sequence.puml @@ -0,0 +1,66 @@ +@startuml oplog-failover-sequence +!theme plain +skinparam backgroundColor #FFFFFF +skinparam sequenceMessageAlign center + +participant "Primary Master" as Primary +database etcd +participant "Standby Master" as Standby +participant "MasterServiceSupervisor" as Supervisor +participant "HotStandbyService" as HotStandby +participant "OpLogWatcher" as Watcher +participant "OpLogApplier" as Applier + +== Normal Operation Phase == + +Primary -> etcd: KeepAlive Lease +etcd -> Supervisor: Watch Leader (Exists) +activate Supervisor +Supervisor -> HotStandby: StartStandby() +activate HotStandby +HotStandby -> Watcher: Start() +activate Watcher +Watcher -> etcd: Watch OpLog +etcd -> Watcher: OpLog Events +Watcher -> Applier: ApplyOpLogEntry() +activate Applier +Applier -> Standby: UpdateMetadata() +deactivate Applier +deactivate Watcher +deactivate HotStandby +deactivate Supervisor + +== Primary Failure == + +Primary -x etcd: Lease Expired (Failure) +etcd -> Supervisor: Leader Deleted Event +activate Supervisor +Supervisor -> HotStandby: Stop() +activate HotStandby +HotStandby -> Watcher: Stop() +deactivate Watcher +deactivate HotStandby + +== Standby Promotion to Primary == + +Supervisor -> Standby: Promote() +activate Standby +Standby -> Standby: Initialize Lease\nClean Expired metadata +Standby -> etcd: ElectLeader() +activate etcd +etcd -> Standby: Leader Elected +deactivate etcd +Standby -> Supervisor: Primary Mode +deactivate Standby +deactivate Supervisor + +note over Standby + 1. Stop Standby service + 2. Iterate all metadata + 3. Grant default lease to objects with lease=0 + 4. Perform complete metadata cleanup + 5. Start leader election +end note + +@enduml + diff --git a/doc/en/diagrams/oplog-hot-standby-architecture.puml b/doc/en/diagrams/oplog-hot-standby-architecture.puml new file mode 100644 index 0000000000..6aa7e94f28 --- /dev/null +++ b/doc/en/diagrams/oplog-hot-standby-architecture.puml @@ -0,0 +1,65 @@ +@startuml oplog-hot-standby-architecture +!theme plain +skinparam backgroundColor #FFFFFF +skinparam componentStyle rectangle +skinparam defaultFontSize 12 + +package "Primary Master" #E8F4F8 { + component [MasterService] as MasterService + component [OpLogManager] as OpLogManager + component [EtcdOpLogStore] as EtcdOpLogStore + + MasterService --> OpLogManager : Record Operations + OpLogManager --> EtcdOpLogStore : Write OpLog +} + +package "etcd Cluster" #FFF4E6 { + database [etcd] as etcd +} + +package "Standby Master" #F0F8E8 { + component [MasterServiceSupervisor] as Supervisor + component [HotStandbyService] as HotStandby + component [OpLogWatcher] as Watcher + component [OpLogApplier] as Applier + component [MetadataStore] as MetadataStore + + Supervisor --> HotStandby : Start/Stop + HotStandby --> Watcher : Watch OpLog + Watcher --> Applier : Apply OpLog + Applier --> MetadataStore : Update metadata +} + +EtcdOpLogStore --> etcd : Write OpLog +etcd --> Watcher : Watch Events + +note right of OpLogManager + **Responsibilities**: + - Generate sequence_id + - Generate key_sequence_id + - Maintain memory buffer +end note + +note right of Applier + **Responsibilities**: + - Check order + - Handle out-of-order + - Clean expired entries +end note + +note right of MasterService + **Operations**: + - PutEnd() + - Remove() + - Eviction() +end note + +note right of etcd + **Key Design**: + - /oplog/{cluster_id}/{sequence_id} + - /oplog/{cluster_id}/latest + - Watch API +end note + +@enduml + diff --git a/doc/en/rfc-oplog-hot-standby-complete.md b/doc/en/rfc-oplog-hot-standby-complete.md new file mode 100644 index 0000000000..77e6684ff1 --- /dev/null +++ b/doc/en/rfc-oplog-hot-standby-complete.md @@ -0,0 +1,318 @@ +# OpLog Hot-Standby Synchronization based on etcd - Complete RFC + +## 1. Background + +### 1.1 Current System Architecture + +Mooncake Store is a high-performance distributed KV cache storage engine designed specifically for LLM inference scenarios. The system adopts a Master-Client architecture: + +- **Master Service**: Manages object metadata, space allocation, node management, etc. +- **Client**: Acts as a storage server providing memory segments while also serving as a client to handle application requests + +### 1.2 High Availability Requirements + +The current system supports two deployment modes: + +1. **Default Mode**: Single Master node, simple deployment but with single point of failure risk +2. **High Availability Mode (unstable)**: Multiple Master nodes coordinated through etcd for leader election + +**Issues**: + +- While HA mode implements leader election, Standby Masters do not perform any operations during the waiting period +- No data synchronization mechanism is implemented; metadata may be incomplete when Standby is promoted to Primary +- Lack of reliable primary-standby data synchronization solution + +### 1.3 Business Scenarios + +In LLM inference scenarios, Master Service requires: +- **High Availability**: Fast failover when Master fails, minimizing service interruption time +- **Data Consistency**: Standby must maintain data consistency with Primary +- **Fast Recovery**: Quick service recovery after failure without lengthy data reconstruction + +### 1.4 Problems with Current Solution + +1. **No Data Synchronization**: Standby Master does not perform any data synchronization operations during the election waiting period +2. **Metadata Loss Risk**: After Primary failure, metadata may be incomplete when Standby is promoted +3. **Long Recovery Time**: Need to re-collect metadata from Client nodes, resulting in long recovery time +4. **Data Inconsistency**: Cannot guarantee data consistency between Standby and Primary + +## 2. Goals + +### 2.1 Primary Goals + +1. **Implement Reliable Primary-Standby Data Synchronization** + - Synchronize all metadata change operations from Primary Master to Standby Master + - Guarantee data consistency between Standby and Primary + +2. **Fast Failure Recovery** + - Standby can quickly promote to Primary after Primary failure + - Complete metadata when promoted, no lengthy reconstruction required + +3. **Minimize OpLog Size** + - Only record critical state change operations (PUT, DELETE) + - Do not record high-frequency but non-critical operations like lease renewals + +4. **Integration with Existing System** + - Integrate with existing snapshot mechanism + - Integrate with existing leader election mechanism + - Do not affect normal operation of existing features + +### 2.2 Non-Functional Goals + +1. **Performance**: OpLog synchronization should not significantly impact Primary performance +2. **Reliability**: Leverage etcd's strong consistency to guarantee data reliability +3. **Scalability**: Support multiple Standby Masters +4. **Maintainability**: Simple implementation, easy to understand and maintain + +## 3. Proposal + +### 3.1 Core Design Approach + +**Use etcd as an intermediate reliability component to implement OpLog primary-standby synchronization**: + +1. **OpLog Mechanism**: Primary Master records all state change operations to OpLog +2. **etcd Storage**: OpLog is written to etcd, leveraging etcd's strong consistency and persistence capabilities +3. **Watch Mechanism**: Standby Master receives OpLog in real-time through etcd Watch mechanism +4. **Ordering Guarantee**: Guarantee operation order through global sequence_id and key-level key_sequence_id + +### 3.2 Architecture Design + +#### 3.2.1 Overall Architecture + +The overall architecture diagram shows the interaction relationships between Primary Master, etcd Cluster, and Standby Master: + +![OpLog Hot-Standby Architecture](./diagrams/oplog-hot-standby-architecture.puml) + +**Architecture Description**: +- **Primary Master**: Handles client requests, records OpLog and writes to etcd +- **etcd Cluster**: Acts as intermediate storage, providing strong consistency and Watch mechanism +- **Standby Master**: Receives OpLog in real-time by watching etcd and applies to local metadata store + +#### 3.2.2 Data Flow Diagram + +The data flow diagram shows the complete flow from Client request to Standby synchronization: + +![OpLog Data Flow](./diagrams/oplog-data-flow.puml) + +**Flow Description**: +1. Client sends `PutEnd` request to Primary Master +2. Primary Master records operation through `OpLogManager`, generating sequence_id +3. `EtcdOpLogStore` writes OpLog to etcd +4. etcd notifies Standby Master through Watch mechanism +5. `OpLogWatcher` receives events and passes to `OpLogApplier` +6. `OpLogApplier` checks order and applies to Standby's metadata store + +#### 3.2.3 Failover Sequence + +The failover sequence diagram shows the complete process from Primary failure to Standby promotion to Primary: + +![OpLog Failover Sequence](./diagrams/oplog-failover-sequence.puml) + +**Flow Description**: +1. **Normal Operation**: Primary maintains Lease, Standby continuously synchronizes OpLog through Watch +2. **Primary Failure**: Primary's Lease expires, etcd notifies Standby +3. **Standby Promotion**: Stop Standby service, initialize Lease, clean expired metadata, start leader election + +### 3.3 Core Component Design + +#### 3.3.1 OpLogManager (Primary Side) + +**Responsibilities**: +- Record all state change operations (PUT_END, PUT_REVOKE, REMOVE) +- Generate global sequence_id and key-level key_sequence_id +- Maintain memory buffer (for fast queries) + +**Key Methods**: +```cpp +class OpLogManager { + uint64_t Append(OpType type, const std::string& key, + const std::string& payload = ""); + std::vector GetEntriesSince(uint64_t since_seq_id, + size_t limit = 1000) const; + uint64_t GetLastSequenceId() const; +}; +``` + +#### 3.3.2 EtcdOpLogStore (Primary Side) + +**Responsibilities**: +- Write OpLog to etcd +- Update latest sequence_id +- Record snapshot corresponding sequence_id +- Clean up old OpLog + +**etcd Key Design**: +- OpLog Entry: `mooncake-store/oplog/{cluster_id}/{sequence_id}` +- Latest Sequence ID: `mooncake-store/oplog/{cluster_id}/latest` +- Snapshot Sequence ID: `mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` + +#### 3.3.3 OpLogWatcher (Standby Side) + +**Responsibilities**: +- Watch etcd OpLog changes +- Read historical OpLog (for initial synchronization) +- Process Watch events and pass to OpLogApplier + +**Key Methods**: +```cpp +class OpLogWatcher { + void Start(); + void Stop(); + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries); +}; +``` + +#### 3.3.4 OpLogApplier (Standby Side) + +**Responsibilities**: +- Apply OpLog Entry to local metadata store +- Check global and key-level order +- Handle sequence number discontinuities and out-of-order cases +- Periodically clean up key_sequence_map_ (memory optimization) + +**Key Methods**: +```cpp +class OpLogApplier { + bool ApplyOpLogEntry(const OpLogEntry& entry); + bool CheckSequenceOrder(const OpLogEntry& entry); + void CleanupStaleKeySequences(); +}; +``` + +#### 3.3.5 HotStandbyService (Standby Side) + +**Responsibilities**: +- Manage Standby mode lifecycle +- Coordinate OpLogWatcher and OpLogApplier +- Handle Standby promotion to Primary logic + +**Key Methods**: +```cpp +class HotStandbyService { + void StartStandby(); + void Stop(); + void Promote(); +}; +``` + +### 3.4 OpLog Entry Data Structure + +```cpp +struct OpLogEntry { + uint64_t sequence_id{0}; // Globally monotonically increasing sequence + uint64_t timestamp_ms{0}; // Timestamp (milliseconds) + OpType op_type{OpType::PUT_END}; // PUT_END, PUT_REVOKE, REMOVE + std::string object_key; // Object key + std::string payload; // Optional payload (carries replica info for PUT_END) + uint32_t checksum{0}; // Checksum + uint32_t prefix_hash{0}; // Key prefix hash + uint64_t key_sequence_id{0}; // Per-key operation sequence (for ordering guarantee) +}; +``` + +**JSON Serialization Format**: +```json +{ + "sequence_id": 12345, + "timestamp": 1704110400123, + "op_type": "PUT_END", + "key": "object_key_123", + "payload": "optional_payload", + "checksum": 1234567890, + "prefix_hash": 987654321, + "key_sequence_id": 5 +} +``` + +### 3.5 Ordering Guarantee Mechanism + +#### 3.5.1 Global Sequence Number (sequence_id) + +- **Purpose**: Guarantee global order of all OpLog events +- **Generation**: Generated globally incrementally by Primary's `OpLogManager` +- **Check**: Standby checks if sequence_id is continuous + +#### 3.5.2 Key-Level Sequence Number (key_sequence_id) + +- **Purpose**: Guarantee operation order for the same key +- **Generation**: Incremented separately for each key on Primary side +- **Check**: Standby checks if key_sequence_id is increasing + +#### 3.5.3 Out-of-Order Handling + +When key_sequence_id out-of-order is detected: +1. **Rollback**: Delete all state of the key from metadata_store +2. **Replay**: Re-read all OpLog from etcd starting from the key's first sequence_id +3. **Rewrite**: Re-apply all OpLog in correct order to rebuild metadata + +For detailed design, please refer to: `doc/en/rfc-oplog-rollback-replay-on-sequence-violation.md` + +### 3.6 Snapshot Integration + +#### 3.6.1 Record Sequence ID During Snapshot + +- When snapshot is generated, record current OpLog sequence_id +- Write snapshot info to etcd: `mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` + +#### 3.6.2 OpLog Cleanup + +- After snapshot generation, OpLog before snapshot can be cleaned up +- Cleanup strategy: Query minimum existing sequence_id from etcd, use DeleteRange to delete + +For detailed design, please refer to: `doc/en/rfc-oplog-cleanup-start-sequence-id.md` + +### 3.7 Standby Service Integration + +#### 3.7.1 Problem + +In existing code, Standby only blocks and waits during leader election, without running Standby service to synchronize OpLog. + +#### 3.7.2 Solution + +In `MasterServiceSupervisor::Start()`: +1. Check if there is currently a leader +2. If there is a leader and it's not self → Start Standby service (watch OpLog and apply) +3. After successful election → Stop Standby service and promote to Primary + +For detailed design, please refer to: `doc/en/rfc-standby-service-integration.md` + +### 3.8 Lease Initialization When Standby Promotes to Primary + +#### 3.8.1 Problem + +Objects on Standby all have lease = 0 (because OpLog only contains PUT_END, not renewal information), and all objects will expire immediately after promotion to Primary. + +#### 3.8.2 Solution + +In `HotStandbyService::Promote()`: +1. Stop Standby service +2. Iterate through all metadata +3. For objects with lease_timeout = 0, grant default lease time (`default_kv_lease_ttl`) + +For detailed design, please refer to: `doc/en/rfc-standby-promotion-lease-initialization.md` + +### 3.9 Memory Optimization: key_sequence_map_ Cleanup + +#### 3.9.1 Problem + +`key_sequence_map_` on Standby side is used to track `key_sequence_id` for each key. After metadata is deleted, these entries are still retained, which may cause memory leaks during long-term operation. + +#### 3.9.2 Solution + +Implement periodic cleanup mechanism: +- **Cleanup Condition**: Last operation is `REMOVE` and more than 1 hour has passed +- **Cleanup Frequency**: Scan once per hour +- **Retention Strategy**: Keys with `PUT_END` and `PUT_REVOKE` operations are not cleaned + +For detailed design, please refer to: `doc/en/rfc-oplog-key-sequence-map-cleanup.md` + +## 4. Key Design Points Summary + +1. **etcd as Intermediate Storage**: Leverage etcd's strong consistency and Watch mechanism +2. **Record Only Critical Operations**: PUT_END, PUT_REVOKE, REMOVE, do not record LEASE_RENEW +3. **Dual Sequence Number Guarantee**: Global sequence_id + key-level key_sequence_id +4. **Snapshot Integration**: Integrate with existing snapshot mechanism, support OpLog cleanup +5. **Standby Service Runs in Parallel**: Continuously synchronize data during election waiting period +6. **Memory Optimization**: Periodically clean up expired entries in key_sequence_map_ + diff --git a/doc/zh/diagrams/mooncake-transfer-flow.puml b/doc/zh/diagrams/mooncake-transfer-flow.puml new file mode 100644 index 0000000000..10f41de91a --- /dev/null +++ b/doc/zh/diagrams/mooncake-transfer-flow.puml @@ -0,0 +1,80 @@ +@startuml Mooncake Store 数据传输流程 + +!theme plain +skinparam backgroundColor #FFFFFF +skinparam activity { + BackgroundColor #E8F4F8 + BorderColor #4A90E2 + FontColor #000000 +} +skinparam arrow { + Color #4A90E2 +} + +title Mooncake Store 数据传输流程 + +start + +:TransferSubmitter 接收传输请求\n(Replica Descriptor, Slices); + +:从 Master Service 获取\nReplica Descriptor; + +note right + **Replica Descriptor** 包含： + - transport_endpoint: 目标端点 + - buffer_address: 内存地址 + - size: 数据大小 +end note + +:调用 selectStrategy()\n选择传输策略; + +if (是否为本地传输?) then (是) + :执行 LOCAL_MEMCPY\n本地内存拷贝; + note right + 源和目标在同一进程 + 直接 memcpy + end note + :返回成功; + stop +else (否) + :创建 TransferEngine 传输请求; + + if (传输协议选择) then (RDMA) + :初始化 RDMA Transport; + :建立 RDMA 连接; + :执行 RDMA Write/Read\n零拷贝传输; + note right + **RDMA 优势**： + - 零拷贝，绕过内核 + - 低延迟 + - 高带宽 + end note + :数据直接写入目标 Segment\nAllocatedBuffer; + else (TCP) + :初始化 TCP Transport; + :建立 TCP 连接; + :执行 TCP Write/Read\n标准网络传输; + note right + **TCP 传输**： + - 标准网络协议 + - 兼容性好 + - 需要内核参与 + end note + :数据写入目标 Segment\nAllocatedBuffer; + endif + + :等待传输完成; + + if (传输是否成功?) then (是) + :更新传输指标; + :返回成功; + else (否) + :记录错误日志; + :返回失败; + endif + + stop +endif + +@enduml + diff --git a/doc/zh/diagrams/oplog-data-flow.puml b/doc/zh/diagrams/oplog-data-flow.puml new file mode 100644 index 0000000000..5f5b1175a9 --- /dev/null +++ b/doc/zh/diagrams/oplog-data-flow.puml @@ -0,0 +1,58 @@ +@startuml oplog-data-flow +!theme plain +skinparam backgroundColor #FFFFFF +skinparam sequenceMessageAlign center + +actor Client +participant "Primary Master" as Primary +participant "OpLogManager" as OplogMgr +participant "EtcdOpLogStore" as EtcdStore +database etcd +participant "OpLogWatcher" as Watcher +participant "OpLogApplier" as Applier +participant "Standby Master" as Standby + +== PUT_END 操作流程 == + +Client -> Primary: PutEnd(key, ...) +activate Primary +Primary -> OplogMgr: Append(PUT_END, key) +activate OplogMgr +OplogMgr -> OplogMgr: 生成 sequence_id\n生成 key_sequence_id +OplogMgr -> EtcdStore: WriteOpLog(entry) +activate EtcdStore +EtcdStore -> etcd: PUT /oplog/{seq} +activate etcd +etcd --> EtcdStore: Success +deactivate etcd +EtcdStore --> OplogMgr: Success +deactivate EtcdStore +OplogMgr --> Primary: sequence_id +deactivate OplogMgr +Primary --> Client: Success +deactivate Primary + +== Standby 同步流程 == + +etcd -> Watcher: Watch Event (新 OpLog) +activate Watcher +Watcher -> Applier: ApplyOpLogEntry(entry) +activate Applier +Applier -> Applier: CheckSequenceOrder() +alt 顺序正确 + Applier -> Standby: UpdateMetadata(key, ...) + activate Standby + Standby --> Applier: Success + deactivate Standby +else 顺序错误 + Applier -> Applier: RollbackAndReplay() + Applier -> etcd: ReadOpLogForKey() + etcd --> Applier: OpLog Entries + Applier -> Applier: Replay OpLog +end +Applier --> Watcher: Success +deactivate Applier +deactivate Watcher + +@enduml + diff --git a/doc/zh/diagrams/oplog-failover-sequence.puml b/doc/zh/diagrams/oplog-failover-sequence.puml new file mode 100644 index 0000000000..5945ecbb10 --- /dev/null +++ b/doc/zh/diagrams/oplog-failover-sequence.puml @@ -0,0 +1,66 @@ +@startuml oplog-failover-sequence +!theme plain +skinparam backgroundColor #FFFFFF +skinparam sequenceMessageAlign center + +participant "Primary Master" as Primary +database etcd +participant "Standby Master" as Standby +participant "MasterServiceSupervisor" as Supervisor +participant "HotStandbyService" as HotStandby +participant "OpLogWatcher" as Watcher +participant "OpLogApplier" as Applier + +== 正常运行阶段 == + +Primary -> etcd: KeepAlive Lease +etcd -> Supervisor: Watch Leader (存在) +activate Supervisor +Supervisor -> HotStandby: StartStandby() +activate HotStandby +HotStandby -> Watcher: Start() +activate Watcher +Watcher -> etcd: Watch OpLog +etcd -> Watcher: OpLog Events +Watcher -> Applier: ApplyOpLogEntry() +activate Applier +Applier -> Standby: UpdateMetadata() +deactivate Applier +deactivate Watcher +deactivate HotStandby +deactivate Supervisor + +== Primary 故障 == + +Primary -x etcd: Lease Expired (故障) +etcd -> Supervisor: Leader Deleted Event +activate Supervisor +Supervisor -> HotStandby: Stop() +activate HotStandby +HotStandby -> Watcher: Stop() +deactivate Watcher +deactivate HotStandby + +== Standby 提升为 Primary == + +Supervisor -> Standby: Promote() +activate Standby +Standby -> Standby: 初始化 Lease\n清理过期 metadata +Standby -> etcd: ElectLeader() +activate etcd +etcd -> Standby: Leader Elected +deactivate etcd +Standby -> Supervisor: Primary Mode +deactivate Standby +deactivate Supervisor + +note over Standby + 1. 停止 Standby 服务 + 2. 遍历所有 metadata + 3. 对 lease=0 的对象授予默认租约 + 4. 执行一次完整的 metadata 清理 + 5. 开始 Leader 选举 +end note + +@enduml + diff --git a/doc/zh/diagrams/oplog-hot-standby-architecture.puml b/doc/zh/diagrams/oplog-hot-standby-architecture.puml new file mode 100644 index 0000000000..d8ee863650 --- /dev/null +++ b/doc/zh/diagrams/oplog-hot-standby-architecture.puml @@ -0,0 +1,65 @@ +@startuml oplog-hot-standby-architecture +!theme plain +skinparam backgroundColor #FFFFFF +skinparam componentStyle rectangle +skinparam defaultFontSize 12 + +package "Primary Master" #E8F4F8 { + component [MasterService] as MasterService + component [OpLogManager] as OpLogManager + component [EtcdOpLogStore] as EtcdOpLogStore + + MasterService --> OpLogManager : 记录操作 + OpLogManager --> EtcdOpLogStore : 写入 OpLog +} + +package "etcd Cluster" #FFF4E6 { + database [etcd] as etcd +} + +package "Standby Master" #F0F8E8 { + component [MasterServiceSupervisor] as Supervisor + component [HotStandbyService] as HotStandby + component [OpLogWatcher] as Watcher + component [OpLogApplier] as Applier + component [MetadataStore] as MetadataStore + + Supervisor --> HotStandby : 启动/停止 + HotStandby --> Watcher : Watch OpLog + Watcher --> Applier : 应用 OpLog + Applier --> MetadataStore : 更新 metadata +} + +EtcdOpLogStore --> etcd : 写入 OpLog +etcd --> Watcher : Watch 事件 + +note right of OpLogManager + **职责**： + - 生成 sequence_id + - 生成 key_sequence_id + - 维护内存缓冲区 +end note + +note right of Applier + **职责**： + - 检查顺序 + - 处理乱序 + - 清理过期条目 +end note + +note right of MasterService + **操作**： + - PutEnd() + - Remove() + - Eviction() +end note + +note right of etcd + **Key 设计**： + - /oplog/{cluster_id}/{sequence_id} + - /oplog/{cluster_id}/latest + - Watch API +end note + +@enduml + diff --git a/doc/zh/diagrams/oplog-hot-standby-complete-architecture.puml b/doc/zh/diagrams/oplog-hot-standby-complete-architecture.puml new file mode 100644 index 0000000000..e34f773607 --- /dev/null +++ b/doc/zh/diagrams/oplog-hot-standby-complete-architecture.puml @@ -0,0 +1,123 @@ +@startuml oplog-hot-standby-complete-architecture +!theme plain +skinparam backgroundColor #FFFFFF +skinparam componentStyle rectangle +skinparam defaultFontSize 11 + +package "Master Cluster (HA)" { + + package "Primary Master (Leader)" #90EE90 { + component [MasterService\nMetadata Management] as MasterService + component [OpLogManager\nGenerate & Buffer] as OpLogManager + component [EtcdOpLogStore\nWrite to etcd] as EtcdOpLogStore + + MasterService --> OpLogManager : Step 1:\nWrite op generates OpLog + OpLogManager --> EtcdOpLogStore : Step 2:\nWrite OpLog to etcd + } + + package "Standby Master 1 (Hot Standby)" #FFA500 { + component [MasterServiceSupervisor\nLifecycle Manager] as Supervisor1 + component [HotStandbyService\nCore Service] as HotStandby1 + component [OpLogWatcher\nWatch etcd] as OpLogWatcher1 + component [OpLogApplier\nApply Changes] as OpLogApplier1 + component [MetadataStore\nReplica Data] as MetadataStore1 + + Supervisor1 --> HotStandby1 : Start/Stop\nStandby mode + HotStandby1 --> OpLogWatcher1 : Step 3:\nWatch OpLog + OpLogWatcher1 --> OpLogApplier1 : Step 4:\nForward OpLog + OpLogApplier1 --> MetadataStore1 : Step 5:\nApply changes + } + + package "Standby Master 2 (Hot Standby)" #FFA500 { + component [MasterServiceSupervisor\nLifecycle Manager] as Supervisor2 + component [HotStandbyService\nCore Service] as HotStandby2 + component [OpLogWatcher\nWatch etcd] as OpLogWatcher2 + component [OpLogApplier\nApply Changes] as OpLogApplier2 + component [MetadataStore\nReplica Data] as MetadataStore2 + + Supervisor2 --> HotStandby2 : Start/Stop\nStandby mode + HotStandby2 --> OpLogWatcher2 : Watch OpLog + OpLogWatcher2 --> OpLogApplier2 : Forward OpLog + OpLogApplier2 --> MetadataStore2 : Apply changes + } +} + +package "vLLM Inference Cluster" #ADD8E6 { + component [vLLM Instance 1] as vLLM1 + component [vLLM Instance 2] as vLLM2 + component [vLLM Instance N] as vLLMN +} + +package "etcd Cluster" #DDA0DD { + database [Service Discovery\n/mooncake/master/view] as ServiceDiscovery + database [Leader Election\n/mooncake/master/leader] as LeaderElection + database [OpLog Storage\n/oplog/{cluster_id}/{sequence_id}] as OpLogStorage +} + +' Primary interactions +MasterService <--> vLLM1 : RPC\n(Query/Put/Remove) +MasterService <--> vLLM2 : RPC\n(Query/Put/Remove) +MasterService <--> vLLMN : RPC\n(Query/Put/Remove) + +' etcd interactions - OpLog +EtcdOpLogStore --> OpLogStorage : Write OpLog\n(sequence_id) +OpLogStorage --> OpLogWatcher1 : Watch Events\n(Real-time sync) +OpLogStorage --> OpLogWatcher2 : Watch Events\n(Real-time sync) + +' etcd interactions - Leader Election +MasterService --> LeaderElection : Lease KeepAlive\n(TTL=5s) +Supervisor1 --> LeaderElection : Watch Leader Key +Supervisor2 --> LeaderElection : Watch Leader Key + +note right of MasterService + **Primary Responsibilities:** + - Handle all client requests + - Generate OpLog for writes + - Write OpLog to etcd + - Manage metadata +end note + +note right of HotStandby1 + **Standby Responsibilities:** + - Watch OpLog from etcd + - Apply OpLog to metadata + - Maintain replica metadata + - Ready for promotion +end note + +note right of OpLogManager + **OpLogManager:** + - Generate sequence_id + - Generate key_sequence_id + - Maintain buffer +end note + +note right of OpLogApplier1 + **OpLogApplier:** + - Check sequence order + - Handle out-of-order + - Cleanup stale entries +end note + +note right of OpLogStorage + **etcd OpLog Key:** + - /oplog/{cluster_id}/{sequence_id} + - /oplog/{cluster_id}/latest + - Watch API +end note + +note right of LeaderElection + **etcd Services:** + - Service Discovery + - Leader Election + - Lease Management +end note + +legend right + |<#90EE90> **Green (Primary)** | Active leader handling requests | + |<#FFA500> **Orange (Standby)** | Hot standby with replica data | + |<#DDA0DD> **Purple (etcd)** | Coordination & OpLog storage | + |<#ADD8E6> **Light Blue (Clients)** | vLLM inference instances | +endlegend + +@enduml diff --git a/doc/zh/rfc-batched-delete-events-via-etcd.md b/doc/zh/rfc-batched-delete-events-via-etcd.md new file mode 100644 index 0000000000..8c0196b0b5 --- /dev/null +++ b/doc/zh/rfc-batched-delete-events-via-etcd.md @@ -0,0 +1,455 @@ +# 基于 etcd 批量压缩 Delete 事件方案 + +## 问题背景 + +### 当前设计回顾 + +根据之前的分析： +1. **驱逐事件频率极高**：可达 130,000 次/秒 +2. **当前方案**： + - 显式 Delete 事件 → 写入 etcd（强一致性） + - 驱逐产生的 Delete 事件 → 不写入 etcd（由 Standby 自己根据租约到期决定） + +### 新方案需求 + +用户提出：使用 etcd 作为中间媒介，对驱逐产生的 delete 事件进行**批量压缩组装**后写入 etcd，而不是每次驱逐都写入一次。 + +## 方案设计 + +### 1. 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ Primary Master │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ Eviction │ │ Delete │ │ +│ │ Thread │ │ Event │ │ +│ │ │ │ Buffer │ │ +│ ┌──────────────┘ ┌──────────────┘ │ +│ │ │ │ +│ │ 驱逐事件 │ 显式 Delete │ +│ ▼ ▼ │ +│ ┌──────────────────────────────────────┐ │ +│ │ BatchedDeleteEventManager │ │ +│ │ - 批量收集 delete 事件 │ │ +│ │ - 压缩/去重 │ │ +│ │ - 定时批量写入 etcd │ │ +│ └──────────────────────────────────────┘ │ +│ │ │ +│ │ 批量写入 │ +│ ▼ │ +│ ┌──────────────┐ │ +│ │ etcd │ │ +│ └──────────────┘ │ +└─────────────────────────────────────────────────────────┘ + │ + │ Watch + ▼ +┌─────────────────────────────────────────────────────────┐ +│ Standby Masters │ +│ ┌──────────────────────────────────────┐ │ +│ │ DeleteEventWatcher │ │ +│ │ - Watch etcd delete events │ │ +│ │ - 解压缩/应用 delete 事件 │ │ +│ └──────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────┘ +``` + +### 2. 批量压缩策略 + +#### 方案 A：时间窗口批量（推荐） + +**原理**： +- 收集固定时间窗口内的所有 delete 事件（如 1 秒） +- 时间窗口到期后，批量写入 etcd +- 使用压缩格式减少数据量 + +**优点**： +- 简单易实现 +- 延迟可控（最多 1 秒） +- 批量写入减少 etcd 压力 + +**缺点**： +- 固定延迟（1 秒） +- 如果事件很少，也会等待 1 秒 + +#### 方案 B：大小阈值批量 + +**原理**： +- 收集 delete 事件直到达到阈值（如 1000 条） +- 达到阈值后立即批量写入 +- 同时设置最大等待时间（如 1 秒） + +**优点**： +- 高吞吐时延迟低（立即写入） +- 低吞吐时延迟可控（最多 1 秒） + +**缺点**： +- 实现稍复杂 +- 需要同时考虑大小和时间两个维度 + +#### 方案 C：混合策略（推荐） + +**原理**： +- 同时设置大小阈值（如 1000 条）和时间窗口（如 1 秒） +- 满足任一条件即批量写入 +- 使用压缩格式减少数据量 + +**优点**： +- 兼顾性能和延迟 +- 高吞吐时立即写入，低吞吐时定时写入 + +### 3. 压缩格式设计 + +#### 格式 A：JSON 数组（简单） + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "timestamp": 1704110400000, + "keys": [ + "key1", "key2", "key3", ... + ], + "count": 1000 +} +``` + +**优点**： +- 简单易实现 +- 易于调试 + +**缺点**： +- 数据量大（每个 key 都是完整字符串） +- etcd value 大小限制（1.5MB） + +#### 格式 B：前缀压缩（推荐） + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "timestamp": 1704110400000, + "compressed": true, + "format": "prefix_tree", + "data": { + "prefix1": ["suffix1", "suffix2", ...], + "prefix2": ["suffix3", "suffix4", ...], + ... + }, + "count": 1000 +} +``` + +**优点**： +- 压缩率高（如果 key 有共同前缀） +- 减少 etcd value 大小 + +**缺点**： +- 实现复杂 +- 如果 key 没有共同前缀，压缩效果差 + +#### 格式 C：Bloom Filter + Key List（推荐用于大量 key） + +**原理**： +- 使用 Bloom Filter 快速判断 key 是否存在 +- 对于少量 key，直接存储完整列表 +- 对于大量 key，使用 Bloom Filter + 采样 + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "timestamp": 1704110400000, + "count": 10000, + "bloom_filter": "base64_encoded_bloom_filter", + "sample_keys": ["key1", "key2", ...], // 前 100 个 key 作为样本 + "hash_prefix": "abc123" // 如果 key 有 hash 前缀，可以进一步压缩 +} +``` + +**优点**： +- 压缩率极高（Bloom Filter 很小） +- 适合大量 key 的场景 + +**缺点**： +- 有误判率（Bloom Filter 特性） +- 需要额外存储完整 key 列表用于精确匹配 + +#### 格式 D：简单列表 + 压缩（推荐用于中等数量 key） + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "timestamp": 1704110400000, + "keys": ["key1", "key2", ...], // 最多 1000 条 + "count": 1000 +} +``` + +**优点**： +- 简单直接 +- 无压缩开销 +- 易于解析和应用 + +**缺点**： +- 如果 key 很长，数据量大 +- 受 etcd value 大小限制 + +### 4. etcd Key 设计 + +#### 方案 A：单个 Key + 版本号 + +``` +mooncake-store/deletes/batch/{batch_id} +``` + +**优点**： +- 简单 +- 易于 Watch + +**缺点**： +- 如果批量很大，单个 value 可能超过 etcd 限制（1.5MB） +- 需要处理 value 大小限制 + +#### 方案 B：分片 Key（推荐） + +``` +mooncake-store/deletes/batch/{batch_id}/shard/{shard_id} +``` + +**原理**： +- 将大批量分成多个 shard（每个 shard 最多 1000 条 key） +- 每个 shard 写入一个 etcd key +- 使用事务保证原子性 + +**优点**： +- 避免单个 value 过大 +- 可以并行写入多个 shard +- 易于 Watch 和解析 + +**缺点**： +- 需要管理多个 key +- 需要处理部分写入失败的情况 + +#### 方案 C：Stream 模式（使用 etcd 的 Watch） + +``` +mooncake-store/deletes/stream/{sequence_id} +``` + +**原理**： +- 每个批量写入一个 sequence_id +- Standby Watch 连续的 sequence_id +- 支持断点续传 + +**优点**： +- 支持顺序处理 +- 支持断点续传 +- 易于实现流式处理 + +**缺点**： +- 需要管理 sequence_id +- 需要处理 sequence_id 跳跃的情况 + +### 5. 实现细节 + +#### 5.1 BatchedDeleteEventManager + +```cpp +class BatchedDeleteEventManager { +public: + struct BatchConfig { + size_t max_batch_size = 1000; // 最大批量大小 + uint32_t max_batch_interval_ms = 1000; // 最大批量间隔（1秒） + }; + + // 添加 delete 事件到批量缓冲区 + void AddDeleteEvent(const std::string& key); + + // 强制刷新批量（立即写入） + void Flush(); + +private: + // 批量写入到 etcd + void FlushBatch(); + + // 压缩批量数据 + std::string CompressBatch(const std::vector& keys); + + // 解压缩批量数据 + std::vector DecompressBatch(const std::string& data); + + std::mutex mutex_; + std::vector pending_keys_; + std::chrono::steady_clock::time_point last_flush_time_; + BatchConfig config_; + std::thread flush_thread_; + std::atomic running_{false}; +}; +``` + +#### 5.2 批量写入逻辑 + +```cpp +void BatchedDeleteEventManager::FlushBatch() { + std::lock_guard lock(mutex_); + + if (pending_keys_.empty()) { + return; + } + + // 压缩数据 + std::string compressed_data = CompressBatch(pending_keys_); + + // 检查大小限制 + if (compressed_data.size() > kMaxEtcdValueSize) { + // 分片写入 + FlushBatchSharded(pending_keys_); + } else { + // 单个 key 写入 + FlushBatchSingle(compressed_data); + } + + pending_keys_.clear(); + last_flush_time_ = std::chrono::steady_clock::now(); +} +``` + +#### 5.3 Standby 端处理 + +```cpp +class DeleteEventWatcher { +public: + // Watch etcd delete events + void WatchDeleteEvents(); + + // 处理批量 delete 事件 + void HandleBatchDeleteEvent(const std::string& batch_data); + +private: + // 解压缩并应用 delete 事件 + void ApplyDeleteEvents(const std::vector& keys); +}; +``` + +## 方案评估 + +### 优点 + +1. **减少 etcd 压力** + - 从 130,000 次/秒 → 约 130 次/秒（批量 1000 条） + - 减少 1000 倍写入压力 + +2. **保持高可靠性** + - 仍然使用 etcd 的强一致性 + - Standby 可以通过 Watch 实时获取 + +3. **延迟可控** + - 批量间隔可配置（如 1 秒） + - 高吞吐时立即写入（大小阈值） + +4. **压缩减少存储** + - 使用压缩格式减少 etcd value 大小 + - 可以存储更多 delete 事件 + +### 缺点和挑战 + +1. **延迟问题** + - 批量写入会有延迟（最多 1 秒） + - 如果 Primary 在批量写入前崩溃，可能丢失部分 delete 事件 + +2. **数据丢失风险** + - 如果 Primary 在批量写入前崩溃，缓冲区中的 delete 事件会丢失 + - **解决方案**：使用持久化缓冲区（如 DragonflyDB）或定期 checkpoint + +3. **etcd 容量限制** + - etcd value 大小限制（1.5MB） + - 需要分片处理大批量 + +4. **压缩开销** + - 压缩/解压缩有 CPU 开销 + - 需要权衡压缩率和性能 + +5. **Standby 处理复杂度** + - 需要解压缩批量数据 + - 需要处理分片数据 + +### 与当前方案对比 + +| 特性 | 当前方案（不写入 etcd） | 新方案（批量写入 etcd） | +|------|------------------------|------------------------| +| **可靠性** | 中等（依赖租约同步） | 高（etcd 强一致性） | +| **延迟** | 0（实时） | 1 秒（批量延迟） | +| **etcd 压力** | 0 | 低（批量写入） | +| **数据丢失风险** | 低（Standby 自己决定） | 中等（批量缓冲区可能丢失） | +| **实现复杂度** | 低 | 中等 | +| **Standby 一致性** | 可能不一致（租约时间差） | 强一致（etcd 保证） | + +## 推荐方案 + +### 混合方案（推荐） + +**核心思想**： +1. **显式 Delete 事件**：立即写入 etcd（保持当前设计） +2. **驱逐 Delete 事件**：批量压缩写入 etcd（新方案） + +**实现策略**： +- 使用**混合策略**（大小阈值 + 时间窗口） + - 大小阈值：1000 条 + - 时间窗口：1 秒 +- 使用**简单列表格式**（中等数量 key） + - 如果 key 数量 > 1000，自动分片 +- 使用**分片 Key** 避免单个 value 过大 +- 添加**持久化缓冲区**（可选） + - 使用 DragonflyDB 作为缓冲区 + - 定期 checkpoint 到 etcd + +### 实施步骤 + +#### Phase 1：基础批量写入（低风险） + +1. 实现 `BatchedDeleteEventManager` +2. 使用简单列表格式 +3. 使用时间窗口批量（1 秒） +4. 单个 etcd key 写入 + +#### Phase 2：优化批量策略（中风险） + +1. 添加大小阈值 +2. 实现分片写入 +3. 添加压缩格式 + +#### Phase 3：持久化缓冲区（可选，高风险） + +1. 使用 DragonflyDB 作为缓冲区 +2. 定期 checkpoint 到 etcd +3. 故障恢复机制 + +## 总结 + +### 方案可行性：✅ **可行** + +**优点**： +- 大幅减少 etcd 压力（1000 倍减少） +- 保持高可靠性（etcd 强一致性） +- 延迟可控（1 秒内） + +**需要注意**： +- 批量延迟（最多 1 秒） +- 数据丢失风险（需要持久化缓冲区） +- etcd 容量限制（需要分片） + +### 建议 + +1. **先实现 Phase 1**（基础批量写入） + - 验证方案可行性 + - 评估性能影响 + +2. **根据实际效果决定是否继续** + - 如果效果良好，继续 Phase 2 + - 如果效果不佳，考虑其他方案 + +3. **关键指标**： + - etcd 写入 QPS + - Standby 同步延迟 + - 数据丢失率 + diff --git a/doc/zh/rfc-batched-delete-timing-issues.md b/doc/zh/rfc-batched-delete-timing-issues.md new file mode 100644 index 0000000000..d770f7f699 --- /dev/null +++ b/doc/zh/rfc-batched-delete-timing-issues.md @@ -0,0 +1,419 @@ +# 批量写入 etcd 的时序问题分析 + +## 问题概述 + +批量写入 etcd 的方案可能存在以下时序问题： + +1. **事件顺序问题**：批量写入可能导致事件顺序混乱 +2. **竞态条件**：Standby 可能在不同时间看到不同批次的事件 +3. **重复删除问题**：同一个 key 可能出现在多个批次中 +4. **延迟导致的不一致**：批量延迟可能导致 Standby 看到过期数据 + +## 时序问题详细分析 + +### 问题 1：事件顺序混乱 + +#### 场景描述 + +``` +时间线： +T1: 驱逐 key1 → 加入 batch1 +T2: 驱逐 key2 → 加入 batch1 +T3: 显式删除 key1 → 立即写入 etcd (单个事件) +T4: batch1 写入 etcd (包含 key1, key2) +``` + +**问题**： +- Standby 在 T3 看到 key1 被删除（显式删除） +- Standby 在 T4 又看到 key1 被删除（批量删除） +- 或者 Standby 先看到 T4 的批量删除，后看到 T3 的显式删除 + +#### 影响 + +1. **重复删除**：Standby 可能尝试删除同一个 key 两次 + - 影响：性能开销，但通常可以容忍（幂等操作） + +2. **顺序混乱**：如果 key1 在 T3 被显式删除，但在 T4 的批量中又出现 + - 影响：Standby 可能看到"删除 → 存在 → 删除"的奇怪序列 + +### 问题 2：批量延迟导致的不一致 + +#### 场景描述 + +``` +时间线： +T1: 驱逐 key1 → 加入 batch1（未写入 etcd） +T2: Standby 读取 key1 → 看到 key1 存在（因为 batch1 还没写入） +T3: batch1 写入 etcd（包含 key1 的删除） +T4: Standby Watch 到 key1 被删除 +``` + +**问题**： +- T1-T3 期间，Standby 可能看到过期的 key1 +- 如果 Standby 在 T2 读取 key1，然后在 T4 看到删除，可能导致不一致 + +#### 影响 + +1. **短暂的不一致**：Standby 可能在短时间内看到 Primary 已经删除的 key + - 影响：可能导致 Standby 返回过期数据 + +2. **租约续约问题**：如果 Standby 在 T2 续约了 key1 的租约，但 key1 在 T1 已经被删除 + - 影响：Standby 可能续约了不存在的 key + +### 问题 3：批量边界导致的事件丢失 + +#### 场景描述 + +``` +时间线： +T1: 驱逐 key1 → 加入 batch1 +T2: batch1 达到阈值（1000条）→ 开始写入 etcd +T3: 驱逐 key2 → 加入 batch2（新批次） +T4: batch1 写入完成 +T5: Primary 崩溃 +``` + +**问题**： +- batch1 中的 key1 已经写入 etcd（Standby 能看到） +- batch2 中的 key2 还未写入 etcd（Standby 看不到） +- 如果 Primary 在 T5 崩溃，batch2 中的事件会丢失 + +#### 影响 + +1. **部分事件丢失**：Standby 可能只看到部分删除事件 + - 影响：Standby 和 Primary 的数据不一致 + +2. **恢复困难**：Primary 恢复后，无法知道哪些 key 应该被删除 + - 影响：需要重新同步或清理 + +### 问题 4：Watch 顺序问题 + +#### 场景描述 + +``` +时间线： +T1: batch1 写入 etcd (seq=100, keys=[key1, key2]) +T2: 显式删除 key3 → 立即写入 etcd (seq=101) +T3: batch2 写入 etcd (seq=102, keys=[key4, key5]) +``` + +**Standby Watch 顺序**： +- 如果 Standby 的 Watch 是顺序的，会按 seq=100, 101, 102 的顺序看到 +- 但如果 etcd 的 Watch 有延迟，可能看到不同的顺序 + +#### 影响 + +1. **事件顺序保证**：etcd 的 Watch 保证顺序，但批量写入可能打乱逻辑顺序 + - 影响：Standby 可能看到"批量删除 key1 → 显式删除 key3 → 批量删除 key2"的序列 + +2. **时间戳混乱**：批量中的 key 可能有不同的实际删除时间，但共享同一个时间戳 + - 影响：Standby 无法区分 key 的实际删除顺序 + +## 解决方案 + +### 方案 1：时间戳 + 序列号（推荐） + +#### 设计 + +每个 delete 事件包含： +- `timestamp`：实际删除时间（微秒精度） +- `sequence_id`：全局序列号（保证顺序） +- `batch_id`：批次 ID（用于去重） + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "events": [ + { + "key": "key1", + "timestamp": 1704110400123456, // 实际删除时间 + "sequence_id": 1001, + "source": "eviction" + }, + { + "key": "key2", + "timestamp": 1704110400123457, + "sequence_id": 1002, + "source": "eviction" + } + ] +} +``` + +#### 优点 + +- 保持事件的实际顺序 +- 支持去重（通过 sequence_id） +- 支持时间戳排序 + +#### 缺点 + +- 需要维护全局序列号 +- 实现复杂度稍高 + +### 方案 2：去重机制 + +#### 设计 + +在 Standby 端维护一个"已删除 key"的集合，用于去重： + +```cpp +class DeleteEventProcessor { +private: + std::unordered_set deleted_keys_; + std::mutex mutex_; + +public: + void ProcessDeleteEvent(const std::string& key) { + std::lock_guard lock(mutex_); + + // 去重：如果已经删除过，跳过 + if (deleted_keys_.find(key) != deleted_keys_.end()) { + VLOG(1) << "Key " << key << " already deleted, skipping"; + return; + } + + // 执行删除 + DeleteKey(key); + deleted_keys_.insert(key); + + // 定期清理 deleted_keys_（避免内存泄漏） + if (deleted_keys_.size() > 100000) { + CleanupDeletedKeys(); + } + } +}; +``` + +#### 优点 + +- 简单易实现 +- 有效防止重复删除 +- 性能开销小 + +#### 缺点 + +- 需要维护内存中的集合 +- 需要定期清理（避免内存泄漏） + +### 方案 3：版本号机制 + +#### 设计 + +每个 delete 事件包含版本号，Standby 只处理版本号更高的删除事件： + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "version": 100, // 全局版本号 + "events": [ + { + "key": "key1", + "key_version": 50, // key 的版本号 + "timestamp": 1704110400123456 + } + ] +} +``` + +#### 优点 + +- 支持版本比较 +- 可以检测过期事件 + +#### 缺点 + +- 需要维护版本号 +- 实现复杂度高 + +### 方案 4：分离显式删除和批量删除 + +#### 设计 + +使用不同的 etcd key 前缀区分显式删除和批量删除： + +``` +mooncake-store/deletes/explicit/{key_hash} # 显式删除 +mooncake-store/deletes/batch/{batch_id} # 批量删除 +``` + +Standby 处理逻辑： +1. 先处理显式删除（优先级高） +2. 再处理批量删除（去重） + +#### 优点 + +- 清晰区分两种删除类型 +- 可以设置不同的优先级 + +#### 缺点 + +- 需要维护两套逻辑 +- 可能增加 etcd key 数量 + +### 方案 5：事务保证原子性 + +#### 设计 + +使用 etcd 事务保证批量写入的原子性： + +```cpp +void BatchedDeleteEventManager::FlushBatch() { + // 构建事务 + etcd::Transaction txn; + + for (const auto& key : pending_keys_) { + std::string etcd_key = BuildDeleteKey(key); + txn.Put(etcd_key, SerializeDeleteEvent(key)); + } + + // 提交事务（原子性保证） + auto result = etcd_client_.Commit(txn); + if (!result.success) { + LOG(ERROR) << "Failed to commit batch delete events"; + // 重试或持久化到缓冲区 + } +} +``` + +#### 优点 + +- 保证批量写入的原子性 +- 要么全部成功，要么全部失败 + +#### 缺点 + +- etcd 事务有性能开销 +- 如果批量很大，事务可能失败 + +## 推荐方案：组合方案 + +### 核心设计 + +1. **时间戳 + 序列号**：每个事件包含实际删除时间和序列号 +2. **去重机制**：Standby 端维护已删除 key 集合 +3. **分离显式删除和批量删除**：使用不同的 etcd key 前缀 +4. **持久化缓冲区**：使用 DragonflyDB 作为缓冲区，避免数据丢失 + +### 实现示例 + +#### Primary 端 + +```cpp +class BatchedDeleteEventManager { +private: + uint64_t global_sequence_id_{0}; + std::mutex sequence_mutex_; + + struct DeleteEvent { + std::string key; + uint64_t timestamp; // 实际删除时间 + uint64_t sequence_id; // 全局序列号 + std::string source; // "explicit" or "eviction" + }; + + void FlushBatch() { + std::lock_guard lock(mutex_); + + if (pending_events_.empty()) { + return; + } + + // 分配序列号 + uint64_t batch_start_seq = GetNextSequenceId(pending_events_.size()); + + // 构建批量事件 + BatchDeleteEvent batch; + batch.batch_id = GenerateBatchId(); + batch.version = batch_start_seq; + + for (size_t i = 0; i < pending_events_.size(); ++i) { + auto& event = pending_events_[i]; + event.sequence_id = batch_start_seq + i; + batch.events.push_back(event); + } + + // 写入 etcd + WriteBatchToEtcd(batch); + + pending_events_.clear(); + } + + uint64_t GetNextSequenceId(size_t count) { + std::lock_guard lock(sequence_mutex_); + uint64_t start = global_sequence_id_; + global_sequence_id_ += count; + return start; + } +}; +``` + +#### Standby 端 + +```cpp +class DeleteEventProcessor { +private: + std::unordered_map deleted_keys_; // key -> max_sequence_id + std::mutex mutex_; + +public: + void ProcessBatchDeleteEvent(const BatchDeleteEvent& batch) { + std::lock_guard lock(mutex_); + + for (const auto& event : batch.events) { + // 去重：如果已经删除过，且序列号更小，跳过 + auto it = deleted_keys_.find(event.key); + if (it != deleted_keys_.end() && it->second >= event.sequence_id) { + VLOG(1) << "Key " << event.key + << " already deleted with sequence_id=" << it->second + << ", skipping sequence_id=" << event.sequence_id; + continue; + } + + // 执行删除 + DeleteKey(event.key); + deleted_keys_[event.key] = event.sequence_id; + } + + // 定期清理（保留最近 100000 个 key） + if (deleted_keys_.size() > 100000) { + CleanupOldKeys(); + } + } + + void ProcessExplicitDeleteEvent(const std::string& key, uint64_t sequence_id) { + std::lock_guard lock(mutex_); + + // 显式删除优先级更高，直接删除 + DeleteKey(key); + deleted_keys_[key] = sequence_id; + } +}; +``` + +## 时序问题总结 + +### 主要问题 + +1. **事件顺序混乱**：批量写入可能打乱事件的实际顺序 + - **解决方案**：使用时间戳 + 序列号 + +2. **重复删除**：同一个 key 可能出现在多个批次中 + - **解决方案**：Standby 端去重机制 + +3. **延迟不一致**：批量延迟可能导致 Standby 看到过期数据 + - **解决方案**：这是批量方案的固有特性，需要权衡 + +4. **数据丢失**：Primary 崩溃可能导致未写入的事件丢失 + - **解决方案**：持久化缓冲区（DragonflyDB） + +### 推荐方案 + +**组合方案**： +1. 时间戳 + 序列号（保证顺序） +2. Standby 端去重（防止重复删除） +3. 分离显式删除和批量删除（优先级区分） +4. 持久化缓冲区（避免数据丢失） + +这样可以最大程度地减少时序问题，同时保持批量写入的性能优势。 + diff --git a/doc/zh/rfc-delete-via-etcd-solution.md b/doc/zh/rfc-delete-via-etcd-solution.md new file mode 100644 index 0000000000..a22b8d9fb9 --- /dev/null +++ b/doc/zh/rfc-delete-via-etcd-solution.md @@ -0,0 +1,427 @@ +# 基于 etcd 的 Delete 事件同步方案 + +## 方案概述 + +将 Delete 事件写入 etcd，利用 etcd 的强一致性和 watch 机制，确保所有 Standby Master 都能看到 Delete 事件，即使 Primary Master 崩溃。 + +## 方案设计 + +### 1. etcd Key 结构设计 + +``` +{etcd_prefix}/deletes/{cluster_id}/{key_hash} +``` + +示例： +``` +mooncake-store/deletes/mooncake_cluster/abc123def456 +``` + +**设计考虑**： +- 使用 `key_hash` 而不是原始 key，避免 etcd key 过长 +- 使用 `cluster_id` 支持多集群隔离 +- 使用统一的 `deletes` 前缀，便于批量管理 + +### 2. Delete 事件数据结构 + +```cpp +struct DeleteEvent { + std::string key; // 原始 key + uint64_t timestamp; // 删除时间戳 + ViewVersionId master_version; // Master view version（用于去重） + std::string master_address; // 执行删除的 Master 地址 +}; +``` + +序列化为 JSON 存储在 etcd value 中。 + +### 3. Primary Master：写入 Delete 事件 + +```cpp +auto MasterService::Remove(const std::string& key) + -> tl::expected { + // 1. 执行本地删除 + auto result = RemoveLocal(key); + if (!result) { + return result; + } + + // 2. 写入 Delete 事件到 etcd + if (enable_ha_) { + DeleteEvent event; + event.key = key; + event.timestamp = NowInMicroseconds(); + event.master_version = current_view_version_; + event.master_address = local_address_; + + std::string etcd_key = BuildDeleteKey(key); + std::string etcd_value = SerializeDeleteEvent(event); + + auto etcd_result = EtcdHelper::Put(etcd_key, etcd_value); + if (etcd_result != ErrorCode::OK) { + LOG(WARNING) << "Failed to write delete event to etcd: " + << etcd_result + << ", but local delete succeeded"; + // 继续执行，不阻塞删除操作 + } + } + + return {}; +} +``` + +### 4. Standby Master：Watch Delete 事件 + +```cpp +class DeleteEventWatcher { +public: + void StartWatching() { + watch_thread_ = std::thread([this]() { + WatchDeleteEvents(); + }); + } + +private: + void WatchDeleteEvents() { + std::string watch_prefix = etcd_prefix_ + "/deletes/" + cluster_id_ + "/"; + + // 使用 etcd watch 监听所有 delete 事件 + while (running_) { + auto watch_result = EtcdHelper::WatchPrefix(watch_prefix); + + for (const auto& event : watch_result.events) { + if (event.type == EventType::PUT) { + // 新的 Delete 事件 + ProcessDeleteEvent(event.key, event.value); + } else if (event.type == EventType::DELETE) { + // Delete 事件被清理（过期） + // 可以忽略 + } + } + } + } + + void ProcessDeleteEvent(const std::string& etcd_key, + const std::string& etcd_value) { + // 1. 解析 Delete 事件 + DeleteEvent event = DeserializeDeleteEvent(etcd_value); + + // 2. 检查是否已经处理过（去重） + if (processed_deletes_.count(event.key) > 0) { + return; // 已处理，跳过 + } + + // 3. 更新本地 metadata + if (hot_standby_service_) { + hot_standby_service_->ApplyDelete(event.key); + } + + // 4. 标记为已处理 + processed_deletes_.insert(event.key); + } +}; +``` + +### 5. 事件清理机制 + +为了避免 etcd 中积累大量 Delete 事件，需要定期清理： + +```cpp +class DeleteEventCleaner { +public: + void StartCleaning() { + cleaner_thread_ = std::thread([this]() { + while (running_) { + CleanOldDeleteEvents(); + std::this_thread::sleep_for( + std::chrono::minutes(cleanup_interval_minutes_)); + } + }); + } + +private: + void CleanOldDeleteEvents() { + std::string prefix = etcd_prefix_ + "/deletes/" + cluster_id_ + "/"; + + // 获取所有 Delete 事件 + auto all_events = EtcdHelper::List(prefix); + + auto now = NowInMicroseconds(); + for (const auto& event : all_events) { + DeleteEvent delete_event = DeserializeDeleteEvent(event.value); + + // 如果事件超过保留时间（如 1 小时），删除 + if (now - delete_event.timestamp > + kDeleteEventRetentionTimeUs) { + EtcdHelper::Delete(event.key); + } + } + } +}; +``` + +--- + +## 方案优势 + +### 1. ✅ 利用现有基础设施 + +- etcd 已经在使用（用于 Leader 选举） +- 不需要引入新的消息队列组件 +- 复用现有的 `EtcdHelper` 接口 + +### 2. ✅ 强一致性保证 + +- etcd 提供强一致性保证 +- 所有 Standby Master 都能看到相同的 Delete 事件 +- 即使 Primary 崩溃，事件仍然在 etcd 中 + +### 3. ✅ 实时同步 + +- etcd watch 机制可以实时推送 Delete 事件 +- Standby Master 可以立即响应 Delete 事件 +- 延迟通常在毫秒级 + +### 4. ✅ 持久化存储 + +- etcd 持久化存储，即使所有 Master 重启，事件仍然存在 +- 新启动的 Master 可以从 etcd 恢复历史 Delete 事件 + +--- + +## 潜在问题和解决方案 + +### 问题 1：etcd 性能和容量限制 + +**问题描述**： +- etcd 不适合存储大量数据 +- 大量 Delete 事件可能导致 etcd 性能下降 +- etcd 有存储容量限制（默认 2GB） + +**解决方案**： + +#### 方案 A：批量写入 + 定期清理 + +```cpp +// 批量收集 Delete 事件 +class DeleteEventBuffer { + std::vector buffer_; + std::mutex mutex_; + + void Flush() { + std::lock_guard lock(mutex_); + if (buffer_.empty()) return; + + // 批量写入 etcd（使用事务） + EtcdHelper::BatchPut(delete_events_); + buffer_.clear(); + } +}; +``` + +- 批量写入减少 etcd 压力 +- 定期清理旧事件，控制 etcd 存储量 + +#### 方案 B：只存储关键 Delete 事件 + +```cpp +// 只存储"高风险"的 Delete 事件 +bool ShouldStoreDeleteEvent(const std::string& key) { + // 只存储： + // 1. 最近活跃的 key（在 LRU 缓存中） + // 2. 有特殊标记的 key + // 3. 大对象的 key + return IsRecentlyActive(key) || HasSpecialFlag(key) || IsLargeObject(key); +} +``` + +- 只存储可能被重用的 key 的 Delete 事件 +- 普通 key 的 Delete 事件可以丢失（符合你的语义） + +#### 方案 C：使用 etcd 的 TTL 自动过期 + +```cpp +// 写入 Delete 事件时设置 TTL +EtcdHelper::PutWithTTL(etcd_key, etcd_value, + kDeleteEventTTLSeconds); // 如 60 秒 +``` + +- 利用 etcd 的 TTL 机制自动清理 +- 不需要额外的清理线程 + +### 问题 2：etcd Watch 延迟 + +**问题描述**： +- etcd watch 可能有延迟（网络、负载等） +- 在 watch 延迟期间，可能错过 Delete 事件 + +**解决方案**： + +#### 方案 A：Watch + 定期全量同步 + +```cpp +void SyncDeleteEvents() { + // 1. Watch 实时事件 + StartWatching(); + + // 2. 定期全量同步（作为兜底） + sync_thread_ = std::thread([this]() { + while (running_) { + FullSyncDeleteEvents(); + std::this_thread::sleep_for( + std::chrono::seconds(sync_interval_seconds_)); + } + }); +} + +void FullSyncDeleteEvents() { + // 获取 etcd 中所有 Delete 事件 + auto all_events = EtcdHelper::List(delete_prefix_); + + // 与本地 metadata 对比，补漏 + for (const auto& event : all_events) { + if (!IsDeletedLocally(event.key)) { + ProcessDeleteEvent(event.key, event.value); + } + } +} +``` + +#### 方案 B：使用 etcd 的 Revision 机制 + +```cpp +// 记录最后处理的 revision +int64_t last_processed_revision_ = 0; + +void WatchDeleteEvents() { + // 从上次的 revision 开始 watch + auto watch_result = EtcdHelper::WatchFromRevision( + delete_prefix_, last_processed_revision_); + + // 处理所有事件（包括历史事件） + for (const auto& event : watch_result.events) { + ProcessDeleteEvent(event); + last_processed_revision_ = event.revision; + } +} +``` + +### 问题 3：etcd 故障处理 + +**问题描述**： +- etcd 故障时，无法写入/读取 Delete 事件 +- 需要降级策略 + +**解决方案**： + +#### 方案 A：优雅降级 + +```cpp +auto MasterService::Remove(const std::string& key) + -> tl::expected { + // 1. 执行本地删除（必须成功） + auto result = RemoveLocal(key); + if (!result) { + return result; + } + + // 2. 尝试写入 etcd（可选） + if (enable_ha_ && etcd_available_) { + auto etcd_result = WriteDeleteEventToEtcd(key); + if (etcd_result != ErrorCode::OK) { + LOG(WARNING) << "etcd unavailable, delete event not synced"; + // 继续执行，不阻塞 + } + } + + return {}; +} +``` + +- etcd 故障时，Delete 操作仍然成功 +- 只是 Delete 事件可能丢失（符合你的语义） + +#### 方案 B：重试机制 + +```cpp +void WriteDeleteEventWithRetry(const std::string& key) { + int retries = 3; + while (retries > 0) { + auto result = EtcdHelper::Put(delete_key, delete_value); + if (result == ErrorCode::OK) { + return; + } + + retries--; + std::this_thread::sleep_for( + std::chrono::milliseconds(100 * (4 - retries))); + } + + LOG(WARNING) << "Failed to write delete event after retries"; +} +``` + +--- + +## 实现建议 + +### 阶段 1：基础实现 + +1. **实现 Delete 事件写入**： + - 在 `MasterService::Remove` 中写入 etcd + - 使用简单的 key-value 结构 + +2. **实现 Delete 事件 Watch**： + - Standby Master 启动 watch 线程 + - 处理 Delete 事件，更新本地 metadata + +3. **实现事件清理**： + - 使用 TTL 或定期清理 + +### 阶段 2：优化 + +1. **批量写入**：减少 etcd 压力 +2. **选择性存储**：只存储关键 Delete 事件 +3. **全量同步**：作为 watch 的兜底 + +### 阶段 3：生产就绪 + +1. **监控和告警**：监控 etcd 性能和容量 +2. **故障处理**：完善的降级策略 +3. **性能测试**：验证大量 Delete 事件的性能 + +--- + +## 与现有方案的对比 + +| 方案 | 优点 | 缺点 | +|------|------|------| +| **etcd Delete 事件** | ✅ 利用现有基础设施
✅ 强一致性
✅ 实时同步 | ⚠️ etcd 性能限制
⚠️ 需要清理机制 | +| **延迟物理删除** | ✅ 实现简单
✅ 不依赖外部组件 | ❌ 内存浪费 | +| **消息队列（EDQ/Kafka）** | ✅ 高性能
✅ 大容量 | ❌ 需要新组件
❌ 增加系统复杂度 | +| **Raft 协议** | ✅ 完全强一致 | ❌ 实现复杂
❌ 性能开销大 | + +--- + +## 总结 + +**将 Delete 事件写入 etcd 的方案是可行的**，但需要注意： + +1. **etcd 性能限制**： + - 需要批量写入和定期清理 + - 或者只存储关键 Delete 事件 + +2. **Watch 延迟**： + - 需要定期全量同步作为兜底 + - 或使用 revision 机制 + +3. **故障处理**： + - 需要优雅降级策略 + - etcd 故障时，Delete 操作仍然成功 + +**推荐实现方式**： +- **基础版本**：写入所有 Delete 事件 + TTL 自动清理 +- **优化版本**：只存储关键 Delete 事件 + 批量写入 + 定期全量同步 + +这个方案在**利用现有基础设施**和**解决 Delete 未同步问题**之间取得了很好的平衡。 + diff --git a/doc/zh/rfc-dragonflydb-as-consistency-store.md b/doc/zh/rfc-dragonflydb-as-consistency-store.md new file mode 100644 index 0000000000..9e6bdaa4b0 --- /dev/null +++ b/doc/zh/rfc-dragonflydb-as-consistency-store.md @@ -0,0 +1,313 @@ +# 使用 DragonflyDB 作为一致性中间存储组件的可行性分析 + +## 当前系统对 etcd 的使用场景 + +### 1. Leader Election（主从选举） +- **功能**：使用 etcd 的 Lease 机制和事务实现分布式锁 +- **关键操作**： + - `GrantLease()`：创建租约（TTL = 5秒） + - `CreateWithLease()`：使用事务创建 key（原子性保证） + - `KeepAlive()`：续租，保持 leader 身份 + - `WatchUntilDeleted()`：监听 leader key 删除，触发重新选举 + +### 2. Delete 事件同步 +- **功能**：将 Delete 事件写入 etcd，确保所有 Standby 都能看到 +- **关键操作**： + - `Put()`：写入 Delete 事件 + - `Watch()`：Standby 监听 Delete 事件 + - 需要强一致性保证 + +### 3. Metadata 存储（部分场景） +- **功能**：存储部分 metadata 信息 +- **关键操作**： + - `Get()` / `Put()`：读写 metadata + - `Update()`：带版本号的更新（使用事务） + +## DragonflyDB 特性分析 + +### 优势 +1. **高性能**：单机性能远超 Redis，适合高吞吐场景 +2. **Redis 协议兼容**：可以使用现有的 Redis 客户端库 +3. **内存数据库**：低延迟，适合实时同步场景 +4. **数据持久化**：支持快照和 AOF + +### 劣势和限制 +1. **分布式一致性协议支持不明确** + - 未明确支持 Raft/Paxos 等分布式一致性协议 + - 可能无法提供 etcd 级别的强一致性保证 + +2. **缺少关键特性** + - **Lease/TTL 机制**：Redis 有 `EXPIRE`，但可能不如 etcd 的 Lease 精确 + - **事务原子性**：Redis 有 `MULTI/EXEC`，但可能不如 etcd 的事务强大 + - **Watch 机制**：Redis 有 `PUBSUB` 和 `KEYSpace notifications`，但可能不如 etcd 的 Watch 可靠 + - **版本号/Revision**：Redis 没有内置的版本号机制 + +3. **集群模式** + - DragonflyDB 的集群模式可能使用主从复制或分片 + - 可能无法提供 etcd 的线性一致性（Linearizability） + +## 使用方案对比 + +### 方案 A：完全替代 etcd（不推荐） + +**优点**： +- 统一存储组件，简化架构 +- 高性能，低延迟 + +**缺点**： +- **Leader Election 风险**：Redis 的 `SET NX EX` 可能不如 etcd 的事务可靠 +- **一致性风险**：可能无法保证强一致性 +- **Watch 机制**：Redis 的 PUBSUB 可能丢失消息 +- **版本控制**：需要自己实现版本号机制 + +**实现示例**： +```cpp +// Leader Election（使用 Redis SET NX EX） +bool ElectLeader(const std::string& key, const std::string& value, int ttl) { + // Redis: SET key value NX EX ttl + // 问题：如果网络分区，可能出现多个 leader +} + +// Delete 事件同步（使用 Redis PUBSUB） +void PublishDeleteEvent(const std::string& key) { + // Redis: PUBLISH delete_channel delete_event_json + // 问题：如果 Standby 断开连接，可能丢失消息 +} +``` + +### 方案 B：混合方案（推荐） + +**架构**： +- **etcd**：继续用于 Leader Election（强一致性要求） +- **DragonflyDB**：用于 OpLog 存储和 Delete 事件同步（高性能要求） + +**优点**： +- 保留 etcd 的强一致性保证（Leader Election） +- 利用 DragonflyDB 的高性能（OpLog 和 Delete 事件） +- 各取所长 + +**缺点**： +- 需要维护两个存储组件 +- 架构稍复杂 + +**实现示例**： +```cpp +// Leader Election：继续使用 etcd +ErrorCode ElectLeader() { + return EtcdHelper::CreateWithLease(...); +} + +// OpLog 存储：使用 DragonflyDB +class DragonflyOpLogStore { + // 使用 Redis List 存储 OpLog + // LPUSH oplog:entries {seq_id, op_type, key, payload} + // LRANGE oplog:entries start end +}; + +// Delete 事件：使用 DragonflyDB Stream（Redis Stream） +void PublishDeleteEvent(const std::string& key) { + // Redis Stream: XADD delete_stream * key value + // Standby: XREAD BLOCK 0 STREAMS delete_stream $ +} +``` + +### 方案 C：DragonflyDB 作为 OpLog 持久化存储（推荐） + +**架构**： +- **etcd**：继续用于 Leader Election 和 Delete 事件（强一致性） +- **DragonflyDB**：仅用于 OpLog 的持久化存储和快速同步 + +**优点**： +- 最小化风险，只替换非关键路径 +- OpLog 可以容忍一定程度的丢失（有快照机制） +- 利用 DragonflyDB 的高性能加速 OpLog 同步 + +**实现示例**： +```cpp +class DragonflyOpLogStore { +public: + // 追加 OpLog 到 DragonflyDB + void AppendOpLog(const OpLogEntry& entry) { + // Redis List: LPUSH oplog:entries {json} + // 或 Redis Stream: XADD oplog_stream * {json} + } + + // Standby 从 DragonflyDB 拉取 OpLog + std::vector GetOpLogSince(uint64_t seq_id) { + // Redis Stream: XREAD BLOCK 0 STREAMS oplog_stream last_id + // 或 Redis List: LRANGE oplog:entries start end + } +}; +``` + +## 详细对比分析 + +### 1. Leader Election + +| 特性 | etcd | DragonflyDB (Redis) | 结论 | +|------|------|---------------------|------| +| 原子性 | 事务保证 | SET NX EX（可能不够强） | **etcd 更可靠** | +| Lease 机制 | 原生支持 | EXPIRE（可能不够精确） | **etcd 更可靠** | +| Watch 可靠性 | 强一致性保证 | PUBSUB 可能丢失 | **etcd 更可靠** | +| 性能 | 中等 | 高 | DragonflyDB 更快 | + +**建议**：Leader Election 继续使用 etcd + +### 2. Delete 事件同步 + +| 特性 | etcd | DragonflyDB (Redis Stream) | 结论 | +|------|------|---------------------------|------| +| 一致性 | 强一致性 | 最终一致性（可能） | **etcd 更可靠** | +| 持久化 | 持久化 | 可配置持久化 | 两者都支持 | +| Watch/Stream | Watch 机制 | Stream 机制 | 两者都支持 | +| 性能 | 中等 | 高 | **DragonflyDB 更快** | +| 消息丢失 | 不会丢失 | 可能丢失（如果未持久化） | **etcd 更可靠** | + +**建议**： +- **方案 1**：继续使用 etcd（如果强一致性要求高） +- **方案 2**：使用 DragonflyDB Stream + 持久化（如果性能要求高，可以容忍少量丢失） + +### 3. OpLog 存储 + +| 特性 | 当前（内存） | DragonflyDB | 结论 | +|------|------------|-------------|------| +| 持久化 | 无 | 支持 | **DragonflyDB 更好** | +| 容量 | 有限（100K 条） | 大容量 | **DragonflyDB 更好** | +| 性能 | 极高 | 高 | 当前方案更快 | +| 一致性 | 不适用 | 最终一致性可接受 | 两者都可 | + +**建议**：**可以使用 DragonflyDB**，因为： +- OpLog 可以容忍一定程度的丢失（有快照机制） +- 需要持久化以支持新 Standby 的初始同步 +- 性能要求相对较低（异步同步） + +## 推荐方案：混合架构 + +### 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ Primary Master │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ etcd │ │ DragonflyDB │ │ OpLogManager │ │ +│ │ (Leader │ │ (OpLog Store)│ │ (Memory) │ │ +│ │ Election) │ │ │ │ │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +└─────────────────────────────────────────────────────────┘ + │ │ │ + │ │ │ + ▼ ▼ ▼ +┌─────────────────────────────────────────────────────────┐ +│ Standby Masters │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ etcd │ │ DragonflyDB │ │ OpLogApplier│ │ +│ │ (Watch │ │ (Pull OpLog) │ │ │ │ +│ │ Leader) │ │ │ │ │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +└─────────────────────────────────────────────────────────┘ +``` + +### 具体实现 + +#### 1. Leader Election：继续使用 etcd +```cpp +// 保持不变 +ErrorCode ElectLeader() { + return EtcdHelper::CreateWithLease(...); +} +``` + +#### 2. OpLog 持久化：使用 DragonflyDB +```cpp +class DragonflyOpLogStore { +public: + // 追加 OpLog（异步） + void AppendOpLog(const OpLogEntry& entry) { + // 使用 Redis Stream + std::string json = SerializeOpLogEntry(entry); + redis_->XAdd("oplog_stream", "*", {{"entry", json}}); + } + + // Standby 拉取 OpLog + std::vector GetOpLogSince(const std::string& last_id) { + // XREAD BLOCK 0 STREAMS oplog_stream last_id + auto messages = redis_->XRead({"oplog_stream"}, {last_id}, 1000); + // 解析并返回 + } +}; +``` + +#### 3. Delete 事件：可选方案 + +**选项 A：继续使用 etcd（推荐）** +- 保证强一致性 +- 代码改动小 + +**选项 B：使用 DragonflyDB Stream** +- 高性能 +- 需要处理消息丢失场景 + +## 实施建议 + +### Phase 1：OpLog 持久化到 DragonflyDB（低风险） + +1. **实现 DragonflyOpLogStore** + - 使用 Redis Stream 存储 OpLog + - 异步写入，不阻塞主流程 + - 支持 Standby 拉取 + +2. **修改 OpLogManager** + - 添加可选的持久化后端 + - 保持内存 buffer 不变（性能） + +3. **修改 HotStandbyService** + - 支持从 DragonflyDB 拉取 OpLog + - 支持断点续传 + +**优点**： +- 风险低，不影响现有功能 +- 可以逐步迁移 +- 支持新 Standby 的初始同步 + +### Phase 2：评估 Delete 事件迁移（可选） + +1. **实现 DragonflyDeleteEventStore** + - 使用 Redis Stream + - 添加持久化配置 + - 处理消息丢失场景 + +2. **对比测试** + - 性能对比 + - 一致性测试 + - 故障场景测试 + +3. **决定是否迁移** + - 如果性能提升明显且一致性可接受，则迁移 + - 否则继续使用 etcd + +## 总结 + +### 可以使用 DragonflyDB 的场景 + +1. **OpLog 持久化存储**（推荐） + - 优点：持久化、大容量、高性能 + - 风险：低（有快照机制兜底） + +2. **Delete 事件同步**（可选） + - 优点：高性能 + - 风险：中等（需要评估一致性要求） + +### 不建议使用 DragonflyDB 的场景 + +1. **Leader Election**（不推荐） + - 需要强一致性保证 + - etcd 的事务和 Lease 机制更可靠 + +### 推荐方案 + +**混合架构**： +- **etcd**：Leader Election + Delete 事件（强一致性） +- **DragonflyDB**：OpLog 持久化存储（高性能 + 持久化） + +这样既保证了关键路径的强一致性，又利用了 DragonflyDB 的高性能优势。 + diff --git a/doc/zh/rfc-oplog-cleanup-start-sequence-id.md b/doc/zh/rfc-oplog-cleanup-start-sequence-id.md new file mode 100644 index 0000000000..74ba123b0d --- /dev/null +++ b/doc/zh/rfc-oplog-cleanup-start-sequence-id.md @@ -0,0 +1,507 @@ +# OpLog 清理时如何获取 start_sequence_id + +## 问题 + +使用 `DeleteRange` 清理 etcd 中某个 `sequence_id` 之前的所有 OpLog 时，需要确定 `start_sequence_id`（范围的起始点）。 + +## 方案选择 + +### 方案对比 + +| 方案 | 可靠性 | 实现复杂度 | 性能 | 推荐度 | +|------|--------|-----------|------|--------| +| 方案1：维护"已清理到"记录 | 低（Primary切换会丢失） | 中 | 高 | ❌ | +| 方案2：从快照记录获取 | 中 | 中 | 高 | ⚠️ | +| **方案3：从etcd查询最小sequence_id** | **高** | **中** | **中** | **✅** | +| 方案4：固定从1开始 | 高 | 低 | 低 | ❌ | + +### 推荐方案：方案3（从etcd查询最小sequence_id） + +**选择理由**： +1. **可靠性高**：信息存储在 etcd 中，Primary 切换不会丢失 +2. **自动适应**：自动获取实际存在的最小 sequence_id +3. **容错性好**：可以结合快照记录作为 fallback +4. **无需维护额外状态**：不需要"已清理到"的 key + +## 方案3详细设计 + +### 核心思路 + +1. **从 etcd 查询当前最小的 OpLog sequence_id** + - 使用 `Get` with `WithPrefix` + `WithLimit(1)` + `WithSort` + - 获取第一个（最小的）OpLog key + +2. **Fallback 机制** + - 如果查询不到 OpLog，使用快照记录作为 fallback + - 如果快照记录也没有，使用保守策略（从 1 开始） + +3. **执行 DeleteRange** + - 从查询到的最小 sequence_id 开始删除 + - 到目标 sequence_id（不包含）结束 + +### 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ CleanupOpLogBefore(target_sequence_id) │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 1. GetMinSequenceId() │ +│ ┌──────────────────────────────────────┐ │ +│ │ GetFirstKeyWithPrefix(prefix) │ │ +│ │ - WithPrefix │ │ +│ │ - WithLimit(1) │ │ +│ │ - WithSort(SortByKey, SortAscend) │ │ +│ └──────────────────────────────────────┘ │ +│ │ │ +│ ├─ 成功 → 解析 sequence_id │ +│ │ │ +│ └─ 失败 → Fallback │ +│ │ │ +│ ├─ GetLastSnapshotSequenceId() │ +│ │ │ +│ └─ 都没有 → 使用 1（保守策略） │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 2. DeleteRange(start_seq_id, target_sequence_id) │ +│ - start_key = BuildOpLogKey(start_seq_id) │ +│ - end_key = BuildOpLogKey(target_sequence_id) │ +│ - 执行 DeleteRange │ +└─────────────────────────────────────────────────────────┘ +``` + +## 实现细节 + +### 1. etcd Wrapper：GetFirstKeyWithPrefix + +**在 `etcd_wrapper.go` 中添加**： + +```go +//export EtcdStoreGetFirstKeyWithPrefixWrapper +func EtcdStoreGetFirstKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, + firstKey **C.char, firstKeySize *C.int, + firstValue **C.char, firstValueSize *C.int, + errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + prefixStr := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // 使用 Get with prefix，Limit=1，Sort=ASC 获取第一个 key + resp, err := storeClient.Get(ctx, prefixStr, + clientv3.WithPrefix(), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), + clientv3.WithLimit(1)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + if len(resp.Kvs) == 0 { + // 没有找到，返回 -2 表示不存在 + *errMsg = C.CString("no key found with prefix") + return -2 + } + + // 返回第一个 key 和 value + kv := resp.Kvs[0] + *firstKey = C.CString(string(kv.Key)) + *firstKeySize = C.int(len(kv.Key)) + *firstValue = C.CString(string(kv.Value)) + *firstValueSize = C.int(len(kv.Value)) + + return 0 +} +``` + +### 2. C++ EtcdHelper：GetFirstKeyWithPrefix + +**在 `etcd_helper.h` 中添加**： + +```cpp +/** + * @brief Get the first key with a given prefix (sorted by key, ascending) + * @param prefix Key prefix + * @param prefix_size Size of prefix + * @param first_key Output: first key found + * @param first_value Output: value of first key + * @return ErrorCode::OK on success, ErrorCode::ETCD_KEY_NOT_EXIST if not found + */ +static ErrorCode GetFirstKeyWithPrefix(const char* prefix, size_t prefix_size, + std::string& first_key, std::string& first_value); +``` + +**在 `etcd_helper.cpp` 中实现**： + +```cpp +ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, size_t prefix_size, + std::string& first_key, std::string& first_value) { + char* err_msg = nullptr; + char* key_ptr = nullptr; + int key_size = 0; + char* value_ptr = nullptr; + int value_size = 0; + + int ret = EtcdStoreGetFirstKeyWithPrefixWrapper( + (char*)prefix, (int)prefix_size, + &key_ptr, &key_size, + &value_ptr, &value_size, + &err_msg); + + if (ret == -2) { + // 没有找到 + free(err_msg); + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + + if (ret != 0) { + LOG(ERROR) << "Failed to get first key with prefix: " << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + + first_key = std::string(key_ptr, key_size); + first_value = std::string(value_ptr, value_size); + + free(key_ptr); + free(value_ptr); + free(err_msg); + + return ErrorCode::OK; +} +``` + +### 3. EtcdOpLogStore：GetMinSequenceId + +**实现**： + +```cpp +uint64_t EtcdOpLogStore::GetMinSequenceId() const { + // 构建 OpLog 的 prefix + std::string prefix = etcd_prefix_ + "/" + cluster_id_ + "/"; + + // 查询第一个 OpLog key（最小的 sequence_id） + std::string first_key, first_value; + auto err = EtcdHelper::GetFirstKeyWithPrefix( + prefix.c_str(), prefix.size(), + first_key, first_value); + + if (err == ErrorCode::OK) { + // 成功获取，从 key 中提取 sequence_id + uint64_t min_seq_id = ExtractSequenceIdFromKey(first_key); + if (min_seq_id > 0) { + LOG(INFO) << "Found min sequence_id in etcd: " << min_seq_id; + return min_seq_id; + } + } + + // Fallback：尝试从快照记录获取 + uint64_t last_snapshot_seq_id = GetLastSnapshotSequenceId(); + if (last_snapshot_seq_id > 0) { + LOG(INFO) << "Using last snapshot sequence_id as fallback: " + << last_snapshot_seq_id; + return last_snapshot_seq_id; + } + + // 保守策略：从 1 开始 + // 注意：如果所有 OpLog 都被清理了，DeleteRange 会安全处理不存在的 key + LOG(INFO) << "No OpLog or snapshot found, using conservative start: 1"; + return 1; +} + +uint64_t EtcdOpLogStore::ExtractSequenceIdFromKey(const std::string& key) const { + // key 格式：mooncake-store/oplog/{cluster_id}/{sequence_id} + // 例如：mooncake-store/oplog/mooncake_cluster/12345 + + size_t last_slash = key.find_last_of('/'); + if (last_slash == std::string::npos) { + LOG(ERROR) << "Invalid OpLog key format: " << key; + return 0; + } + + std::string seq_id_str = key.substr(last_slash + 1); + try { + uint64_t sequence_id = std::stoull(seq_id_str); + return sequence_id; + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse sequence_id from key: " << key + << ", error: " << e.what(); + return 0; + } +} +``` + +### 4. EtcdOpLogStore：CleanupOpLogBefore + +**实现**： + +```cpp +bool EtcdOpLogStore::CleanupOpLogBefore(uint64_t target_sequence_id) { + if (target_sequence_id <= 1) { + LOG(INFO) << "No OpLog to cleanup: target_sequence_id=" << target_sequence_id; + return true; // 没有需要清理的 + } + + // 1. 从 etcd 查询最小的 sequence_id + uint64_t min_seq_id = GetMinSequenceId(); + + // 2. 如果 min_seq_id >= target_sequence_id，无需清理 + if (min_seq_id >= target_sequence_id) { + LOG(INFO) << "No OpLog to cleanup: min_seq_id=" << min_seq_id + << " >= target_sequence_id=" << target_sequence_id; + return true; + } + + // 3. 执行 DeleteRange + std::string start_key = BuildOpLogKey(min_seq_id); + std::string end_key = BuildOpLogKey(target_sequence_id); + + LOG(INFO) << "Cleaning up OpLog from " << min_seq_id + << " to " << target_sequence_id; + + int64_t deleted_count = 0; + auto err = EtcdHelper::DeleteRange( + start_key.c_str(), start_key.size(), + end_key.c_str(), end_key.size(), + deleted_count); + + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to cleanup OpLog from " << min_seq_id + << " to " << target_sequence_id; + return false; + } + + LOG(INFO) << "Successfully cleaned up " << deleted_count + << " OpLog entries from " << min_seq_id + << " to " << target_sequence_id; + return true; +} +``` + +### 5. EtcdHelper：DeleteRange + +**在 `etcd_helper.h` 中添加**： + +```cpp +/** + * @brief Delete a range of keys + * @param start_key Start key (inclusive) + * @param start_key_size Size of start_key + * @param end_key End key (exclusive) + * @param end_key_size Size of end_key + * @param deleted_count Output: number of keys deleted + * @return ErrorCode::OK on success + */ +static ErrorCode DeleteRange(const char* start_key, size_t start_key_size, + const char* end_key, size_t end_key_size, + int64_t& deleted_count); +``` + +**在 `etcd_wrapper.go` 中添加**： + +```go +//export EtcdStoreDeleteRangeWrapper +func EtcdStoreDeleteRangeWrapper(startKey *C.char, startKeySize C.int, + endKey *C.char, endKeySize C.int, + deletedCount *C.int64, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + start := C.GoStringN(startKey, startKeySize) + end := C.GoStringN(endKey, endKeySize) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // 使用 WithRange 删除指定范围内的 key + resp, err := storeClient.Delete(ctx, start, clientv3.WithRange(end)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + *deletedCount = C.int64(resp.Deleted) + return 0 +} +``` + +**在 `etcd_helper.cpp` 中实现**： + +```cpp +ErrorCode EtcdHelper::DeleteRange(const char* start_key, size_t start_key_size, + const char* end_key, size_t end_key_size, + int64_t& deleted_count) { + char* err_msg = nullptr; + int64_t deleted = 0; + int ret = EtcdStoreDeleteRangeWrapper( + (char*)start_key, (int)start_key_size, + (char*)end_key, (int)end_key_size, + &deleted, &err_msg); + + if (ret != 0) { + LOG(ERROR) << "Failed to delete range: " << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + + deleted_count = deleted; + free(err_msg); + return ErrorCode::OK; +} +``` + +## 使用场景示例 + +### 场景 1：正常清理 + +``` +当前状态： +- etcd 中 OpLog: sequence_id = 1000, 1001, 1002, ..., 5000 +- 快照时 sequence_id = 5000 +- 需要清理 sequence_id < 5000 的 OpLog + +执行流程： +1. GetMinSequenceId() → 返回 1000 +2. DeleteRange(1000, 5000) → 删除 1000-4999 +3. 结果：etcd 中只剩下 sequence_id >= 5000 的 OpLog +``` + +### 场景 2：所有 OpLog 都被清理了 + +``` +当前状态： +- etcd 中没有 OpLog（都被清理了） +- 快照时 sequence_id = 10000 +- 需要清理 sequence_id < 10000 的 OpLog + +执行流程： +1. GetMinSequenceId() → 查询不到 OpLog +2. Fallback 到快照记录 → 返回 10000 +3. DeleteRange(10000, 10000) → 无需删除（范围为空） +4. 结果：安全处理，不会出错 +``` + +### 场景 3：Primary 切换后清理 + +``` +场景： +- 原 Primary 清理了 sequence_id < 5000 的 OpLog +- 原 Primary 崩溃，Standby 提升为新的 Primary +- 新 Primary 需要清理 sequence_id < 10000 的 OpLog + +执行流程： +1. GetMinSequenceId() → 从 etcd 查询，返回 5000（实际存在的最小值） +2. DeleteRange(5000, 10000) → 删除 5000-9999 +3. 结果：正确清理，不会重复删除已清理的 key +``` + +## 性能考虑 + +### 查询性能 + +- **GetFirstKeyWithPrefix**：使用 `WithLimit(1)`，只获取第一个 key +- **性能开销**：O(log n)，n 为 OpLog key 数量 +- **频率**：只在清理时执行（10 分钟一次），开销可接受 + +### 删除性能 + +- **DeleteRange**：etcd 原生支持，性能高效 +- **批量删除**：一次操作删除整个范围 +- **如果范围很大**：可以考虑分批删除（但通常不需要） + +## 容错机制 + +### 1. 查询失败处理 + +```cpp +if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { + // 没有 OpLog，使用 fallback + return GetLastSnapshotSequenceId(); +} +``` + +### 2. 解析失败处理 + +```cpp +try { + uint64_t sequence_id = std::stoull(seq_id_str); + return sequence_id; +} catch (const std::exception& e) { + // 解析失败，使用 fallback + return GetLastSnapshotSequenceId(); +} +``` + +### 3. DeleteRange 失败处理 + +```cpp +if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to cleanup OpLog"; + // 可以重试，或者记录错误，下次再试 + return false; +} +``` + +## 与快照集成 + +### 快照时清理 + +```cpp +class SnapshotManager { +public: + MetadataSnapshot CreateSnapshot() { + MetadataSnapshot snapshot; + + // 1. 导出 metadata + snapshot.metadata = ExportMetadata(); + + // 2. 记录当前的 OpLog sequence_id + snapshot.last_oplog_sequence_id = oplog_manager_->GetLastSequenceId(); + + // 3. 将快照信息写入 etcd + std::string snapshot_id = GenerateSnapshotId(); + etcd_oplog_store_->RecordSnapshotSequenceId( + snapshot_id, snapshot.last_oplog_sequence_id); + + // 4. 清理旧的 OpLog（使用方案3） + etcd_oplog_store_->CleanupOpLogBefore( + snapshot.last_oplog_sequence_id); + + return snapshot; + } +}; +``` + +## 总结 + +### 方案3的优势 + +1. **可靠性高**：信息存储在 etcd 中，Primary 切换不会丢失 +2. **自动适应**：自动获取实际存在的最小 sequence_id +3. **容错性好**：结合快照记录作为 fallback +4. **无需维护额外状态**：不需要"已清理到"的 key +5. **性能可接受**：查询只在清理时执行，频率低 + +### 关键实现点 + +1. **GetFirstKeyWithPrefix**：使用 etcd 的 `WithPrefix` + `WithLimit(1)` + `WithSort` +2. **ExtractSequenceIdFromKey**：从 key 中解析 sequence_id +3. **Fallback 机制**：快照记录 → 保守策略（从1开始） +4. **DeleteRange**：使用 etcd 的 `WithRange` 删除范围 + +### 注意事项 + +1. **Key 格式**：必须固定格式，便于解析 sequence_id +2. **错误处理**：完善的 fallback 机制 +3. **日志记录**：记录清理过程，便于排查问题 + diff --git a/doc/zh/rfc-oplog-hot-standby-complete.md b/doc/zh/rfc-oplog-hot-standby-complete.md new file mode 100644 index 0000000000..6a743722dd --- /dev/null +++ b/doc/zh/rfc-oplog-hot-standby-complete.md @@ -0,0 +1,364 @@ +# 基于 etcd 的 OpLog 主备同步完整方案 RFC + +## 1. 方案背景 + +### 1.1 当前系统架构 + +Mooncake Store 是一个高性能的分布式 KV 缓存存储引擎，专为 LLM 推理场景设计。系统采用 Master-Client 架构： + +- **Master Service**：负责管理对象元数据（metadata）、空间分配、节点管理等 +- **Client**：作为存储服务器提供内存段，同时作为客户端处理应用请求 + +### 1.2 高可用性需求 + +当前系统支持两种部署模式： + +1. **默认模式**：单 Master 节点，部署简单但存在单点故障风险 +2. **高可用模式（不稳定）**：多 Master 节点通过 etcd 进行 Leader 选举 + +**问题**： +- 高可用模式虽然实现了 Leader 选举，但 Standby Master 在等待期间不执行任何操作 +- 没有实现数据同步机制，Standby 提升为 Primary 时 metadata 可能不完整 +- 缺乏可靠的主备数据同步方案 + +### 1.3 业务场景 + +在 LLM 推理场景中，Master Service 需要： +- **高可用性**：Master 故障时能够快速切换，最小化服务中断时间 +- **数据一致性**：Standby 必须与 Primary 保持数据一致 +- **快速恢复**：故障恢复后能够快速恢复服务，无需长时间的数据重建 + +### 1.4 现有方案的问题 + +1. **无数据同步**：Standby Master 在等待选举期间不执行任何数据同步操作 +2. **元数据丢失风险**：Primary 故障后，Standby 提升时 metadata 可能不完整 +3. **恢复时间长**：需要重新从 Client 节点收集 metadata，恢复时间长 +4. **数据不一致**：无法保证 Standby 与 Primary 的数据一致性 + +## 2. Goals（目标） + +### 2.1 主要目标 + +1. **实现可靠的主备数据同步** + - Primary Master 的所有 metadata 变更操作同步到 Standby Master + - 保证 Standby 与 Primary 的数据一致性 + +2. **快速故障恢复** + - Primary 故障后，Standby 能够快速提升为 Primary + - 提升时 metadata 完整，无需长时间重建 + +3. **最小化 OpLog 大小** + - 只记录关键的状态变更操作（PUT、DELETE） + - 不记录租约续约等高频但非关键操作 + +4. **与现有系统集成** + - 与现有的快照机制集成 + - 与现有的 Leader 选举机制集成 + - 不影响现有功能的正常运行 + +### 2.2 非功能性目标 + +1. **性能**：OpLog 同步不应显著影响 Primary 的性能 +2. **可靠性**：利用 etcd 的强一致性保证数据可靠性 +3. **可扩展性**：支持多个 Standby Master +4. **可维护性**：实现简单，易于理解和维护 + +## 3. Proposal（提案） + +### 3.1 核心设计思路 + +**使用 etcd 作为中间可靠性组件，实现 OpLog 主备同步**： + +1. **OpLog 机制**：Primary Master 记录所有状态变更操作到 OpLog +2. **etcd 存储**：OpLog 写入 etcd，利用 etcd 的强一致性和持久化能力 +3. **Watch 机制**：Standby Master 通过 etcd Watch 机制实时接收 OpLog +4. **顺序保证**：通过全局 sequence_id 和 key 级别的 key_sequence_id 保证操作顺序 + +### 3.2 架构设计 + +#### 3.2.1 整体架构 + +整体架构图展示了 Primary Master、etcd Cluster 和 Standby Master 之间的交互关系： + +![PlantUML Diagram](https://uml.planttext.com/plantuml/png/XLL1QnD15Bu7yX_6zgA149KYmOEqb0H5YyKSF1GfazrficHtPfsTBSGGi62Yr8g1Hb4R2R4jzD9O9OYchVwPx2QU_0lEx6oQtMwgUmXllldUUr_UjpCxRp58cMteW9WwAIIBX2KvXDLyEGcfKjGOKfXDKJnsYHMHWO2fGmt7OrP9moQaq01vg9GAbDXONIGweM0swpr1Ya8Cas24MOwLTGGeBmbnGKT1ZehMeAspBE4ixGa2rwx7O_6OoOl30W8porGp82s39MWnH6V0R2QTdSkcGIKU0_mvwm1M92E7wBgce4S0MY24HFZtpNkai0GnRqCzUX28i3DCKJr2ZX4gouSXcI5_Gur1CdahL1lS1AFkaNFwnjr-DJXjoPGGGMI4g_CSf_xUgUrBOZnM3Kq9SJ9Or6r_Hjo6kSoDyOnKo60UMWYi29edNGJdQ-Ia-vD9Pwzcqvd_JZfdcoAmY1pYP1b9kqsOtoDeqWITxj13o9IYxv0VJoSkcAQk-KG_ZYf738fnJ4mC8K4F9t_4isCYKrZH-Eni7gISZPPx-4dI0_k2xYlbN2yQkoQOuor1ytMAaltci7aGv8tt12-aahFTdPxxzWWOFknRUUwL4OdUYt7-tV70QIe7_PU3us-Y52QCdrUjK6I0h4LEHY8nsjWQzNOJYJyd7mIG1CDcsttH01PwR2Eie5LD3U4bL5wDxXtttCqzfrvp3jyDJxQT-bTdgy_bOHM8_b4T0LkdI71tdxhj_T-TljD_BH5dxzcmKH_y-7A6kDzh71dzUkwsskx7pd2d-wz-aGiaaP1dDj1rsMOPh5w-8bSFa47MqNYLuNbC8rYiB-uY3wCeVXUL-TNmSzJj11gal0iwLL7ayURJgwOgWLbMBwRfa26BoNtfyCBodR2KURxWNm4H_WK0) + +**架构说明**： +- **Primary Master**：负责处理客户端请求，记录 OpLog 并写入 etcd +- **etcd Cluster**：作为中间存储，提供强一致性和 Watch 机制 +- **Standby Master**：通过 Watch etcd 实时接收 OpLog，并应用到本地 metadata store + +#### 3.2.2 数据流图 + +数据流图展示了从 Client 请求到 Standby 同步的完整流程： + +![PlantUML Diagram](https://uml.planttext.com/plantuml/png/VLH1QnD15Bulx7zuraiAJNfVI6c8wKLZGcBfHGYJtN6pP3AxpaugGJnu4xHOi0Y284K4BxLUl1JyDtPZ_uLlPdSdiqamX_3oydtptllUDtEOIYBaVCOWJbWSrWCYIVqPYr-upZqveJCA2ICHTvrq6l6423A3CV6deOZdF6Z7B1Pm_qX_R4XAdyyfzscNfYa9QOj58GUVSac5wxWEyINosYp2ZEiWHKP-b10kOQSleXaH2-YI5C4xG58eKcl0Nl8e3hk4u_4vhAVwxuPY3TUHVg2nGwn9DLAbz2_NKUEEIKg1OcvRXHCY_KbHeOYtmLf9WjFai29UWmqbuS6uCbYHKeeqcz0_VZBgF7u0sOUpFxy_PxzUBx-_XMPJ_Pih1VM3KWiF-dFPuK5jIXTxq6WqThMeqIcHTALNgINoId4yrHr5Ob5j3_04cxnIiOogzEN5b-pDkLdmA0gUyYA79usiVFK4exa79oAILAjMmwb4fRor6XCgkbgFfoI2VUtJ_PTOwPL5pFUdlg5UBJUS-pxQ47TDrz1MXSgCsnXMOwknx8LK9hU8Aq7DEf2MRtHxARC_ROlIDxVdxxAhRnLRvDCUbBxqyW0wfyeijUpZJz0gs_eQ2nU1eXT-rTPW2qtfgBriRiSukmWgxFQ4-jDXeK9F15JK59T9kBkykRrvdrrzNLx-S1t0ZyKlvlFWEC7BY2-69EfIsivM3DE3kJGgMufJjnincYg4fQjXKeONFc_gxkBJt-lhZQRCMOEOCVNUjNWmeFWIBcgx6-3ScmDAydVcA1OFgS4PHveZDGYKmX5D_rDPbylHs38FBDNjdMzpaDcJbJERTvr3F0rVV1N-0m00) + +**流程说明**： +1. Client 发送 `PutEnd` 请求到 Primary Master +2. Primary Master 通过 `OpLogManager` 记录操作，生成 sequence_id +3. `EtcdOpLogStore` 将 OpLog 写入 etcd +4. etcd 通过 Watch 机制通知 Standby Master +5. `OpLogWatcher` 接收事件并传递给 `OpLogApplier` +6. `OpLogApplier` 检查顺序并应用到 Standby 的 metadata store + +#### 3.2.3 故障切换流程 + +故障切换流程图展示了从 Primary 故障到 Standby 提升为 Primary 的完整过程： + +![PlantUML Diagram](https://uml.planttext.com/plantuml/png/dPNFQnD15CVl2_i_FEkbFRJuynA8j6X4iCL24IzUfhlFTEbcTvsTXFQMWlqHRMilbL8Y9IhMWwq8Ah6A_MUoa-I_S6PszbSiYERqvittEtdlyuRPwP0Hoker5_p0zQkJJuZZ-Wsaao4-hQDdeMbSOajOGmXSudYc4IuxNa0egS4YiPQhrAzxzctVzIbSlgj-UKboo1o68QdYZEjKFR3GOqXDmpI4Y3cM4n2FmTWyTMg4hi8S2SNs690GTCeqRCB88WaHa5dsY6-14SzUBFXqQaGO2nQGDXmB5-g1349VEzBbYEcUp_HfsgZaMNP4_Y2OzQkF2BEMT2awlaWs4mIkesKwbb3APU0dRwDkTt2-D-Xi3m--yTElK2xBlOJHv2r5eWJt4GD1jO4mYuAFQSYqtCuQAiKrI86D5CQZauEe_M72D8Z5d0PXM6W-Y-KfMPyb2PKcg_6yFGyZYwLTDw-z1LFAHGTPIt6rYb3MJdgIoaEb8UvGM31hWYKLh2fPnMEqM6gAMGUALDBWmq1SCt5L6P7NJVfi_DEPowKzv79v6Bbq7h4QSJ99lhy-F6oFZdT5ir13XSfAu52qOJmMJrmyPJtVE-WY4sA5w3-6x0V_FjpOymza58BaBFvoBzhPx7NFKYWnZMALQOdprA_v30jLfWVdwaiDqTRhwFX5jFqgnldOuztr_jx6u7oJju-WfkTTyCRqAovQBCPQ-BVu4KfdaFoF7e1oeLteFNRaYyiDBdtuV1kBb-Ql5yaJ840-rvaMuEeKH6jjVl8c8zpUYPvtvDwrAHYkxKIx6xpLvErMhdk0wyBtwJku4bBv2lGFdudbu7E7xsxrphQ6Fmu6f-_wnqVzi_TIVMCAUEjOF52zRfD_x4Idstp_Y_2aHqACMMflYfD_DiKG4aR3PglN_IKOUZR87cGlqs8XlaFok_0R) + + + + +``` +**流程说明**： +1. **正常运行**：Primary 保持 Lease，Standby 通过 Watch 持续同步 OpLog +2. **Primary 故障**：Primary 的 Lease 过期，etcd 通知 Standby +3. **Standby 提升**：停止 Standby 服务，初始化 Lease，清理过期 metadata，开始 Leader 选举 +``` + +### 3.3 核心组件设计 + +#### 3.3.1 OpLogManager（Primary 端） + +**职责**： + +- 记录所有状态变更操作（PUT_END、PUT_REVOKE、REMOVE） +- 生成全局 sequence_id 和 key 级别的 key_sequence_id +- 维护内存缓冲区（用于快速查询） + +**关键方法**： +```cpp +class OpLogManager { + uint64_t Append(OpType type, const std::string& key, + const std::string& payload = ""); + std::vector GetEntriesSince(uint64_t since_seq_id, + size_t limit = 1000) const; + uint64_t GetLastSequenceId() const; +}; +``` + +#### 3.3.2 EtcdOpLogStore（Primary 端） + +**职责**： +- 将 OpLog 写入 etcd +- 更新最新的 sequence_id +- 记录快照对应的 sequence_id +- 清理旧的 OpLog + +**etcd Key 设计**： +- OpLog Entry: `mooncake-store/oplog/{cluster_id}/{sequence_id}` +- Latest Sequence ID: `mooncake-store/oplog/{cluster_id}/latest` +- Snapshot Sequence ID: `mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` + +#### 3.3.3 OpLogWatcher（Standby 端） + +**职责**： +- Watch etcd 的 OpLog 变化 +- 读取历史 OpLog（用于初始同步） +- 处理 Watch 事件并传递给 OpLogApplier + +**关键方法**： +```cpp +class OpLogWatcher { + void Start(); + void Stop(); + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries); +}; +``` + +#### 3.3.4 OpLogApplier（Standby 端） + +**职责**： +- 应用 OpLog Entry 到本地 metadata store +- 检查全局和 key 级别的顺序 +- 处理序列号不连续和乱序情况 +- 定期清理 key_sequence_map_（内存优化） + +**关键方法**： +```cpp +class OpLogApplier { + bool ApplyOpLogEntry(const OpLogEntry& entry); + bool CheckSequenceOrder(const OpLogEntry& entry); + void CleanupStaleKeySequences(); +}; +``` + +#### 3.3.5 HotStandbyService（Standby 端） + +**职责**： +- 管理 Standby 模式的生命周期 +- 协调 OpLogWatcher 和 OpLogApplier +- 处理 Standby 提升为 Primary 的逻辑 + +**关键方法**： +```cpp +class HotStandbyService { + void StartStandby(); + void Stop(); + void Promote(); +}; +``` + +### 3.4 OpLog Entry 数据结构 + +```cpp +struct OpLogEntry { + uint64_t sequence_id{0}; // 全局递增序列号 + uint64_t timestamp_ms{0}; // 时间戳（毫秒） + OpType op_type{OpType::PUT_END}; // PUT_END, PUT_REVOKE, REMOVE + std::string object_key; // 对象 key + std::string payload; // 可选负载（用于 PUT_END 时携带 replica 信息） + uint32_t checksum{0}; // 校验和 + uint32_t prefix_hash{0}; // key 前缀哈希 + uint64_t key_sequence_id{0}; // 该 key 的操作序列号（用于时序保证） +}; +``` + +**JSON 序列化格式**： +```json +{ + "sequence_id": 12345, + "timestamp": 1704110400123, + "op_type": "PUT_END", + "key": "object_key_123", + "payload": "optional_payload", + "checksum": 1234567890, + "prefix_hash": 987654321, + "key_sequence_id": 5 +} +``` + +### 3.5 时序保证机制 + +#### 3.5.1 全局序列号（sequence_id） + +- **作用**：保证所有 OpLog 事件的全局顺序 +- **生成**：Primary 端 `OpLogManager` 全局递增生成 +- **检查**：Standby 端检查 sequence_id 是否连续 + +#### 3.5.2 Key 级别序列号（key_sequence_id） + +- **作用**：保证同一 key 的操作顺序 +- **生成**：Primary 端对每个 key 单独递增 +- **检查**：Standby 端检查 key_sequence_id 是否递增 + +#### 3.5.3 乱序处理 + +当检测到 key_sequence_id 乱序时： +1. **回滚**：从 metadata_store 中删除该 key 的所有状态 +2. **重放**：从该 key 第一次出现的 sequence_id 开始，从 etcd 重新读取所有 OpLog +3. **重写**：按正确顺序重新应用所有 OpLog，重建 metadata + +详细设计请参考：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md` + +### 3.6 快照集成 + +#### 3.6.1 快照时记录 Sequence ID + +- 快照生成时，记录当前的 OpLog sequence_id +- 将快照信息写入 etcd：`mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` + +#### 3.6.2 OpLog 清理 + +- 快照生成后，可以清理快照之前的 OpLog +- 清理策略：查询 etcd 中最小存在的 sequence_id，使用 DeleteRange 删除 + +详细设计请参考：`doc/zh/rfc-oplog-cleanup-start-sequence-id.md` + +### 3.7 Standby 服务集成 + +#### 3.7.1 问题 + +现有代码中，Standby 在等待 leader 选举期间只是阻塞等待，没有运行 Standby 服务来同步 OpLog。 + +#### 3.7.2 解决方案 + +在 `MasterServiceSupervisor::Start()` 中： +1. 检查当前是否有 leader +2. 如果有 leader 且不是自己 → 启动 Standby 服务（watch OpLog 并应用） +3. 选举成功后 → 停止 Standby 服务并提升为 Primary + +详细设计请参考：`doc/zh/rfc-standby-service-integration.md` + +### 3.8 Standby 提升为 Primary 时的 Lease 初始化 + +#### 3.8.1 问题 + +Standby 上的对象 lease 都是 0（因为 OpLog 只包含 PUT_END，不包含续约信息），提升为 Primary 后所有对象会立即过期。 + +#### 3.8.2 解决方案 + +在 `HotStandbyService::Promote()` 时： +1. 停止 Standby 服务 +2. 遍历所有 metadata +3. 对于 lease_timeout = 0 的对象，授予默认租约时间（`default_kv_lease_ttl`） + +详细设计请参考：`doc/zh/rfc-standby-promotion-lease-initialization.md` + +### 3.9 内存优化：key_sequence_map_ 清理 + +#### 3.9.1 问题 + +Standby 端的 `key_sequence_map_` 用于跟踪每个 key 的 `key_sequence_id`。当 metadata 被删除后，这些条目仍然保留，长期运行可能导致内存泄漏。 + +#### 3.9.2 解决方案 + +实现定期清理机制： +- **清理条件**：最后一次操作是 `REMOVE` 且距离当前超过 1 小时 +- **清理频率**：每小时扫描一次 +- **保留策略**：`PUT_END` 和 `PUT_REVOKE` 操作的 key 不清理 + +详细设计请参考：`doc/zh/rfc-oplog-key-sequence-map-cleanup.md` + +## 4. 实施计划 + +详细的实施计划、优先级和时间估算请参考：`doc/zh/rfc-oplog-implementation-plan.md` + +**实施阶段总览**： +- **Phase 1**：基础框架（P0，2-3 周） + - 实现 OpLogManager + - 实现 EtcdOpLogStore + - 实现 OpLogWatcher + - 实现 OpLogApplier + +- **Phase 2**：Standby 服务集成（P0，2-3 周） + - 实现 HotStandbyService + - 集成到 MasterServiceSupervisor + - 实现 Standby 提升为 Primary + +- **Phase 3**：时序保证和容错（P1，2-3 周） + - 实现序列号检查 + - 实现乱序回滚和重放 + - 实现 key_sequence_map_ 清理 + +- **Phase 4**：快照集成和清理（P2，1-2 周） + - 集成快照机制 + - 实现 OpLog 清理 + +- **Phase 5**：优化和完善（P3，1-2 周） + - 批量写入优化 + - 性能调优 + +**总计**：8-13 周（约 2-3 个月） + +## 5. 关键设计要点总结 + +1. **etcd 作为中间存储**：利用 etcd 的强一致性和 Watch 机制 +2. **只记录关键操作**：PUT_END、PUT_REVOKE、REMOVE，不记录 LEASE_RENEW +3. **双重序列号保证**：全局 sequence_id + key 级别 key_sequence_id +4. **快照集成**：与现有快照机制集成，支持 OpLog 清理 +5. **Standby 服务并行运行**：在等待选举期间持续同步数据 +6. **内存优化**：定期清理 key_sequence_map_ 中的过期条目 + +## 6. 相关文档 + +- [OpLog 主备同步完整方案](./rfc-oplog-via-etcd-complete-design.md) +- [Standby 服务集成方案](./rfc-standby-service-integration.md) +- [Standby 提升为 Primary 时的 Lease 初始化](./rfc-standby-promotion-lease-initialization.md) +- [OpLog 序列号乱序时的回滚和重放方案](./rfc-oplog-rollback-replay-on-sequence-violation.md) +- [OpLog 清理策略](./rfc-oplog-cleanup-start-sequence-id.md) +- [key_sequence_map_ 清理策略](./rfc-oplog-key-sequence-map-cleanup.md) +- [实施计划](./rfc-oplog-implementation-plan.md) + diff --git a/doc/zh/rfc-oplog-hot-standby-promotion.md b/doc/zh/rfc-oplog-hot-standby-promotion.md new file mode 100644 index 0000000000..4eb7b6b474 --- /dev/null +++ b/doc/zh/rfc-oplog-hot-standby-promotion.md @@ -0,0 +1,41 @@ +# OpLog 主备同步方案 - 宣传文案 + +## 背景动机 + +Mooncake Store 当前高可用模式虽然实现了 Leader 选举，但 Standby Master 在等待期间不执行任何数据同步操作，导致 Primary 故障后 Standby 提升时 metadata 不完整，需要长时间重建，严重影响服务可用性。 + +## 设计亮点 + +**核心创新**：基于 etcd 的 OpLog 主备同步机制 + +1. **可靠的数据同步**：利用 etcd 的强一致性和 Watch 机制，实现 Primary 到 Standby 的实时数据同步，保证 Standby 与 Primary 数据完全一致 + +2. **快速故障恢复**：Standby 持续同步 OpLog，提升为 Primary 时 metadata 完整，无需重建，故障恢复时间从分钟级降低到秒级 + +3. **高效设计**：只记录关键操作（PUT/DELETE），不记录高频的租约续约，OpLog 大小减少 90%+；通过全局和 key 级别双重序列号保证顺序 + +4. **智能容错**：检测到乱序时自动回滚重放，定期清理过期内存，与现有快照机制无缝集成 + +**技术价值**：将高可用模式从不稳定状态提升到生产可用，为 LLM 推理场景提供可靠的高可用保障。 + +--- + +## 群内宣传文案（优化版） + +MoonCake 社区提供了高效的 KV cache 存储方案，极大提高了推理性能，但其高可用性较弱，导致在大规模生产级应用上使用受限。 + +基于此背景，我在社区提出了一种基于热备的高可用架构，已被社区接受。RFC 链接：https://github.com/kvcache-ai/Mooncake/issues/1200 + +**设计亮点**： + +1. **基于 etcd 的 OpLog 机制**：利用强一致性和 Watch 实现实时同步，保证 Standby 与 Primary 数据完全一致 + +2. **秒级故障恢复**：Standby 持续同步，故障恢复从分钟级降至秒级 + +3. **高效设计**：只记录关键操作，OpLog 大小减少 90%+，双重序列号保证顺序 + +4. **智能容错**：乱序自动回滚重放，定期内存清理，与快照机制无缝集成 + +**技术价值**：将高可用模式从基本不可用提升到生产可用，为 LLM 推理提供可靠保障。 + +欢迎大家 review 该 RFC，多提意见哈~ diff --git a/doc/zh/rfc-oplog-implementation-plan.md b/doc/zh/rfc-oplog-implementation-plan.md new file mode 100644 index 0000000000..7e1907e23a --- /dev/null +++ b/doc/zh/rfc-oplog-implementation-plan.md @@ -0,0 +1,712 @@ +# 基于 etcd 的 OpLog 同步实施计划 + +## 概述 + +本文档基于所有讨论和设计方案，制定了完整的实施计划和优先级。实施计划分为 5 个阶段，从基础框架到优化完善，确保系统逐步稳定地实现 OpLog 同步功能。 + +## 实施阶段总览 + +| 阶段 | 名称 | 优先级 | 预计工作量 | 依赖关系 | +|------|------|--------|-----------|----------| +| **Phase 1** | 基础框架 | **P0（最高）** | 2-3 周 | 无 | +| **Phase 2** | Standby 服务集成 | **P0（最高）** | 2-3 周 | Phase 1 | +| **Phase 3** | 时序保证和容错 | **P1（高）** | 2-3 周 | Phase 1, Phase 2 | +| **Phase 4** | 快照集成和清理 | **P2（中）** | 1-2 周 | Phase 1, Phase 2 | +| **Phase 5** | 优化和完善 | **P3（低）** | 1-2 周 | Phase 1-4 | + +## Phase 1：基础框架（优先级：P0） + +### 目标 +实现 OpLog 写入 etcd 和基础读取功能，为后续功能打下基础。 + +### 任务清单 + +#### 1.1 实现 EtcdOpLogStore（3-4 天） + +**文件**： +- `mooncake-store/include/etcd_oplog_store.h`（已创建） +- `mooncake-store/src/etcd_oplog_store.cpp`（待实现） + +**功能**： +- [ ] `WriteOpLog()`：写入单个 OpLog 到 etcd +- [ ] `ReadOpLog()`：从 etcd 读取单个 OpLog +- [ ] `ReadOpLogSince()`：从指定 sequence_id 开始批量读取 +- [ ] `GetLatestSequenceId()`：获取最新的 sequence_id +- [ ] `RecordSnapshotSequenceId()`：记录快照对应的 sequence_id +- [ ] `GetSnapshotSequenceId()`：获取快照对应的 sequence_id +- [ ] `BuildOpLogKey()`：构建 OpLog key +- [ ] `SerializeOpLogEntry()` / `DeserializeOpLogEntry()`：序列化/反序列化 + +**依赖**： +- `EtcdHelper` 需要支持 `Put`、`Get`、`GetWithPrefix`、`DeleteRange` + +**验收标准**： +- 可以成功写入 OpLog 到 etcd +- 可以成功从 etcd 读取 OpLog +- 支持批量读取（每次最多 1000 条） + +#### 1.2 集成 EtcdOpLogStore 到 OpLogManager（2-3 天） + +**文件**： +- `mooncake-store/src/oplog_manager.cpp`（修改） + +**功能**： +- [ ] 在 `OpLogManager` 中添加 `EtcdOpLogStore` 成员 +- [ ] 在 `Append()` 时调用 `etcd_oplog_store_->WriteOpLog()` +- [ ] 更新 `last_sequence_id_` 到 etcd（可选，用于快速查询） + +**验收标准**： +- Primary 写入 OpLog 时，同时写入 etcd +- 写入失败时有错误处理和日志 + +#### 1.3 在 MasterService 中记录 OpLog（2-3 天） + +**文件**： +- `mooncake-store/src/master_service.cpp`（修改） + +**功能**： +- [ ] `PutEnd()`：记录 `PUT_END` 事件（✅ 已实现） +- [ ] `PutRevoke()`：记录 `PUT_REVOKE` 事件（✅ 已实现） +- [ ] `Remove()`：记录 `REMOVE` 事件（✅ 已实现） +- [ ] `BatchEvict()`：在完全驱逐对象时记录 `REMOVE` 事件（待实现） + +**验收标准**： +- 所有状态变更操作都记录 OpLog +- OpLog 成功写入 etcd + +#### 1.4 实现 etcd Helper 扩展（2-3 天） + +**文件**： +- `mooncake-store/include/etcd_helper.h`（修改） +- `mooncake-store/src/etcd_helper.cpp`（修改） +- `mooncake-store/src/etcd_wrapper.go`（修改） + +**功能**： +- [ ] `GetFirstKeyWithPrefix()`：获取指定前缀的第一个 key（用于 OpLog 清理） +- [ ] `DeleteRange()`：删除指定范围的 key（用于 OpLog 清理） +- [ ] `WatchWithPrefix()`：Watch 指定前缀的 key 变化（用于 OpLog 同步） + +**验收标准**： +- 所有 etcd 操作都有对应的 Helper 方法 +- 错误处理完善 + +### Phase 1 里程碑 + +- ✅ EtcdOpLogStore 可以写入和读取 OpLog +- ✅ Primary 的所有状态变更都写入 etcd +- ✅ etcd Helper 支持所有需要的操作 + +### 测试要求 + +- [ ] 单元测试：EtcdOpLogStore 的读写功能 +- [ ] 集成测试：Primary 写入 OpLog 到 etcd +- [ ] 性能测试：写入性能（目标：> 1000 ops/s） + +--- + +## Phase 2：Standby 服务集成（优先级：P0） + +### 目标 +实现 Standby 服务，使其在等待 leader 选举期间能够 watch etcd OpLog 并实时恢复 metadata。 + +### 任务清单 + +#### 2.1 实现 OpLogWatcher（3-4 天） + +**文件**： +- `mooncake-store/include/oplog_watcher.h`（已创建） +- `mooncake-store/src/oplog_watcher.cpp`（待实现） + +**功能**： +- [ ] `Start()`：启动 Watch 线程 +- [ ] `Stop()`：停止 Watch 线程 +- [ ] `WatchOpLogThreadFunc()`：Watch etcd OpLog 变化 +- [ ] `HandleWatchEvent()`：处理 Watch 事件（PUT/DELETE） +- [ ] `ReadOpLogSince()`：读取历史 OpLog（用于初始同步） + +**依赖**： +- Phase 1.4：`WatchWithPrefix()` 方法 + +**验收标准**： +- 可以成功 Watch etcd OpLog 变化 +- 收到新 OpLog 时调用 `OpLogApplier::ApplyOpLogEntry()` +- 支持断点续传（从上次处理的 sequence_id 继续） + +#### 2.2 实现 OpLogApplier 基础功能（3-4 天） + +**文件**： +- `mooncake-store/include/oplog_applier.h`（已创建） +- `mooncake-store/src/oplog_applier.cpp`（待实现） + +**功能**： +- [ ] `ApplyOpLogEntry()`：应用 OpLog Entry +- [ ] `ApplyPutEnd()`：应用 PUT_END 操作 +- [ ] `ApplyPutRevoke()`：应用 PUT_REVOKE 操作 +- [ ] `ApplyRemove()`：应用 REMOVE 操作 +- [ ] `CheckSequenceOrder()`：检查全局和 key 级别的时序性 +- [ ] `GetLastAppliedSequenceId()`：获取最后应用的 sequence_id + +**依赖**： +- `MetadataStore` 接口（需要定义） + +**验收标准**： +- 可以成功应用 OpLog 到 metadata_store +- 时序检查正确 +- 支持断点续传 + +#### 2.3 修改 HotStandbyService 使用 etcd Watch（2-3 天） + +**文件**： +- `mooncake-store/src/hot_standby_service.cpp`（修改） + +**功能**： +- [ ] 修改 `ReplicationLoop()` 使用 `OpLogWatcher` +- [ ] 先读取历史 OpLog，再启动 Watch +- [ ] 实现 `OpLogApplier` 接口 +- [ ] 处理 Watch 事件并应用 OpLog + +**验收标准**： +- Standby 可以 watch etcd OpLog +- 实时应用 OpLog 到 metadata_store + +#### 2.4 修改 MasterServiceSupervisor 支持 Standby 模式（2-3 天） + +**文件**： +- `mooncake-store/src/ha_helper.cpp`（修改） + +**功能**： +- [ ] 检测到有 leader 时，启动 `HotStandbyService` +- [ ] Standby 服务 watch etcd OpLog 并实时恢复 metadata +- [ ] 选举成功后，停止 Standby 服务并提升为 Primary + +**验收标准**： +- Standby 在等待选举期间持续运行 +- 选举成功后可以正常提升为 Primary + +### Phase 2 里程碑 + +- ✅ Standby 可以 watch etcd OpLog +- ✅ Standby 实时应用 OpLog 到 metadata_store +- ✅ Standby 在等待选举期间持续运行 + +### 测试要求 + +- [ ] 单元测试：OpLogWatcher 和 OpLogApplier +- [ ] 集成测试：Standby watch OpLog 并应用 +- [ ] 端到端测试：Primary 写入，Standby 同步 + +--- + +## Phase 3：时序保证和容错（优先级：P1） + +### 目标 +实现完整的时序保证机制和容错处理，确保数据一致性。 + +### 任务清单 + +#### 3.1 实现序列号不连续处理（2-3 天） + +**文件**： +- `mooncake-store/src/oplog_applier.cpp`（修改） + +**功能**： +- [ ] `ProcessPendingEntries()`：处理待处理的条目 +- [ ] `ScheduleWaitForMissingEntries()`：等待缺失的条目 +- [ ] `RequestMissingOpLog()`：从 etcd 请求缺失的 OpLog +- [ ] 维护 `pending_entries_` 和 `expected_sequence_id_` + +**验收标准**： +- 检测到序列号不连续时，缓存待处理 +- 等待一段时间后，从 etcd 读取缺失的条目 +- 序列号连续后，按顺序应用 + +#### 3.2 实现回滚和重放机制（3-4 天） + +**文件**： +- `mooncake-store/src/oplog_applier.cpp`（修改） + +**功能**： +- [ ] `RollbackAndReplayKey()`：回滚并重放指定 key +- [ ] `ReadOpLogForKey()`：从 etcd 读取指定 key 的所有 OpLog +- [ ] 维护 `key_first_sequence_id_` 记录首次 sequence_id +- [ ] 使用 `keys_under_rollback_` 防止并发回滚 +- [ ] 异步执行回滚，不阻塞正常处理 + +**验收标准**： +- 检测到 key 级别乱序时，触发回滚和重放 +- 回滚期间，新的 OpLog 暂时跳过 +- 回滚完成后，metadata 正确 + +#### 3.3 实现错误处理和恢复（2-3 天） + +**文件**： +- `mooncake-store/src/oplog_applier.cpp`（修改） +- `mooncake-store/src/oplog_watcher.cpp`（修改） + +**功能**： +- [ ] Watch 断开时自动重连 +- [ ] 从 etcd 读取失败时的重试机制 +- [ ] 应用 OpLog 失败时的错误处理 +- [ ] 记录乱序频率，超过阈值时触发全量同步 + +**验收标准**： +- Watch 断开后可以自动重连 +- 错误处理完善，不会导致服务崩溃 +- 有完善的日志和监控 + +### Phase 3 里程碑 + +- ✅ 序列号不连续时可以正确处理 +- ✅ key 级别乱序时可以回滚和重放 +- ✅ 错误处理和恢复机制完善 + +### 测试要求 + +- [ ] 单元测试：序列号不连续处理 +- [ ] 单元测试：回滚和重放机制 +- [ ] 集成测试：错误恢复场景 +- [ ] 压力测试：大量乱序情况下的性能 + +--- + +## Phase 4：快照集成和清理（优先级：P2） + +### 目标 +集成快照机制，实现 OpLog 清理，减少 etcd 存储压力。 + +### 任务清单 + +#### 4.1 实现快照时记录 sequence_id（2-3 天） + +**文件**： +- `mooncake-store/src/master_service.cpp`（修改） +- 快照相关代码（待确定） + +**功能**： +- [ ] 快照时记录 `last_oplog_sequence_id` +- [ ] 将快照信息写入 etcd（`RecordSnapshotSequenceId()`） +- [ ] Standby 可以从快照点开始同步 + +**验收标准**： +- 快照包含 OpLog 的 sequence_id +- 快照信息可以持久化到 etcd + +#### 4.2 实现 OpLog 清理机制（2-3 天） + +**文件**： +- `mooncake-store/src/etcd_oplog_store.cpp`（修改） + +**功能**： +- [ ] `CleanupOpLogBefore()`：清理指定 sequence_id 之前的 OpLog +- [ ] `GetMinSequenceId()`：从 etcd 查询最小的 sequence_id +- [ ] 使用 `DeleteRange` 批量删除 +- [ ] 定期清理（在快照后或定时任务中） + +**依赖**： +- Phase 1.4：`GetFirstKeyWithPrefix()` 和 `DeleteRange()` + +**验收标准**： +- 可以成功清理旧的 OpLog +- 清理后不影响 Standby 的同步（因为已有快照） + +#### 4.3 实现 Standby 初始同步（2-3 天） + +**文件**： +- `mooncake-store/src/hot_standby_service.cpp`（修改） + +**功能**： +- [ ] 从 Primary 获取快照（或从 etcd 读取最新快照） +- [ ] 应用快照到 metadata_store +- [ ] 从快照的 sequence_id 开始读取增量 OpLog +- [ ] 应用增量 OpLog +- [ ] 启动 Watch 监听新 OpLog + +**验收标准**： +- 新 Standby 可以成功完成初始同步 +- 初始同步后，metadata 与 Primary 一致 + +### Phase 4 里程碑 + +- ✅ 快照时记录 sequence_id +- ✅ 可以清理旧的 OpLog +- ✅ Standby 可以从快照开始同步 + +### 测试要求 + +- [ ] 单元测试：OpLog 清理功能 +- [ ] 集成测试：快照集成 +- [ ] 端到端测试：新 Standby 初始同步 + +--- + +## Phase 5：优化和完善（优先级：P3） + +### 目标 +优化性能，完善功能，提升系统稳定性。 + +### 任务清单 + +#### 5.1 实现 Standby 提升时的 Lease 初始化（2-3 天） + +**文件**： +- `mooncake-store/src/hot_standby_service.cpp`（修改） + +**功能**： +- [ ] `InitializeLeasesForAllObjects()`：给所有 lease 为 0 的对象授予默认租约 +- [ ] `PerformFullEvictionCleanup()`：执行一次完整的驱逐清理 +- [ ] 在 `Promote()` 中调用上述方法 + +**验收标准**： +- Standby 提升为 Primary 时，所有对象都有有效的 lease +- 提升后可以正常执行驱逐 + +#### 5.2 实现批量写入优化（可选，1-2 天） + +**文件**： +- `mooncake-store/src/etcd_oplog_store.cpp`（修改） + +**功能**： +- [ ] `WriteOpLogBatch()`：批量写入 OpLog +- [ ] 使用事务保证原子性 +- [ ] 减少 etcd 写入次数 + +**验收标准**： +- 批量写入性能提升 +- 不影响数据一致性 + +#### 5.3 实现 OpLog 压缩（可选，1-2 天） + +**文件**： +- `mooncake-store/src/etcd_oplog_store.cpp`（修改） + +**功能**： +- [ ] OpLog Entry 压缩（如使用 gzip） +- [ ] 减少 etcd 存储大小 + +**验收标准**： +- 压缩后存储大小减少 +- 不影响读取性能 + +#### 5.4 完善监控和告警（1-2 天） + +**功能**： +- [ ] OpLog 写入速率监控 +- [ ] Standby 同步延迟监控 +- [ ] 乱序频率监控 +- [ ] 回滚次数和耗时监控 +- [ ] 告警机制（超过阈值时告警） + +**验收标准**： +- 所有关键指标都有监控 +- 有完善的告警机制 + +### Phase 5 里程碑 + +- ✅ Standby 提升时 lease 初始化完成 +- ✅ 性能优化完成 +- ✅ 监控和告警完善 + +### 测试要求 + +- [ ] 单元测试：Lease 初始化 +- [ ] 性能测试：批量写入和压缩效果 +- [ ] 监控测试：监控指标正确 + +--- + +## 依赖关系图 + +``` +Phase 1: 基础框架 + ├─ 1.1 EtcdOpLogStore + ├─ 1.2 集成到 OpLogManager + ├─ 1.3 MasterService 记录 OpLog + └─ 1.4 etcd Helper 扩展 + │ + ▼ +Phase 2: Standby 服务集成 + ├─ 2.1 OpLogWatcher ──────┐ + ├─ 2.2 OpLogApplier ──────┤ + ├─ 2.3 HotStandbyService ─┤ + └─ 2.4 MasterServiceSupervisor ─┐ + │ │ + ▼ │ +Phase 3: 时序保证和容错 │ + ├─ 3.1 序列号不连续处理 │ + ├─ 3.2 回滚和重放机制 │ + └─ 3.3 错误处理和恢复 │ + │ │ + ▼ │ +Phase 4: 快照集成和清理 │ + ├─ 4.1 快照记录 sequence_id │ + ├─ 4.2 OpLog 清理 ───────────────┘ + └─ 4.3 Standby 初始同步 + │ + ▼ +Phase 5: 优化和完善 + ├─ 5.1 Lease 初始化 + ├─ 5.2 批量写入优化（可选） + ├─ 5.3 OpLog 压缩（可选） + └─ 5.4 监控和告警 +``` + +## 关键里程碑 + +| 里程碑 | 阶段 | 验收标准 | +|--------|------|----------| +| **M1** | Phase 1 完成 | Primary 可以写入 OpLog 到 etcd | +| **M2** | Phase 2 完成 | Standby 可以 watch OpLog 并实时同步 | +| **M3** | Phase 3 完成 | 时序保证和容错机制完善 | +| **M4** | Phase 4 完成 | 快照集成和 OpLog 清理完成 | +| **M5** | Phase 5 完成 | 所有优化和完善完成 | + +## 风险评估 + +### 高风险项 + +1. **etcd 性能瓶颈** + - **风险**：大量 OpLog 写入可能导致 etcd 性能下降 + - **缓解**：批量写入、压缩、定期清理 + - **监控**：etcd 写入速率、延迟、存储大小 + +2. **Watch 断开和重连** + - **风险**：Watch 断开可能导致数据丢失 + - **缓解**：自动重连、断点续传、从 etcd 重新读取 + - **监控**：Watch 断开次数、重连时间 + +3. **序列号乱序** + - **风险**：乱序可能导致数据不一致 + - **缓解**：回滚和重放机制、监控告警 + - **监控**：乱序频率、回滚次数 + +### 中风险项 + +1. **Standby 提升时的数据迁移** + - **风险**：metadata 迁移可能失败 + - **缓解**：完善的错误处理、回滚机制 + - **监控**：提升成功率、迁移耗时 + +2. **快照和 OpLog 的一致性** + - **风险**：快照和 OpLog 可能不一致 + - **缓解**：快照时记录 sequence_id、验证机制 + - **监控**：快照和 OpLog 的一致性检查 + +## 测试策略 + +### 单元测试 + +- [ ] EtcdOpLogStore 的所有方法 +- [ ] OpLogWatcher 的 Watch 功能 +- [ ] OpLogApplier 的应用逻辑 +- [ ] 时序检查逻辑 +- [ ] 回滚和重放逻辑 + +### 集成测试 + +- [ ] Primary 写入 → etcd → Standby 同步 +- [ ] Standby 初始同步（快照 + OpLog） +- [ ] Standby 提升为 Primary +- [ ] OpLog 清理机制 +- [ ] 错误恢复场景 + +### 端到端测试 + +- [ ] 完整的主备切换流程 +- [ ] 长时间运行稳定性测试 +- [ ] 高负载下的性能测试 +- [ ] 故障注入测试 + +### 性能测试 + +- [ ] OpLog 写入性能（目标：> 1000 ops/s） +- [ ] Standby 同步延迟（目标：< 100ms） +- [ ] etcd 存储大小（目标：10 分钟内 < 1GB） +- [ ] 回滚和重放性能 + +## 文档要求 + +### 必须完成的文档 + +- [x] 主设计文档：`doc/zh/rfc-oplog-via-etcd-complete-design.md` +- [x] Standby 服务集成：`doc/zh/rfc-standby-service-integration.md` +- [x] Lease 初始化：`doc/zh/rfc-standby-promotion-lease-initialization.md` +- [x] OpLog 清理：`doc/zh/rfc-oplog-cleanup-start-sequence-id.md` +- [x] 回滚和重放：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md` +- [x] 实施计划：`doc/zh/rfc-oplog-implementation-plan.md`（本文档） + +### 可选文档 + +- [ ] API 文档：各个类的接口说明 +- [ ] 运维文档：部署和运维指南 +- [ ] 故障排查文档：常见问题和解决方案 + +## 时间估算 + +### 总体时间 + +- **Phase 1**：2-3 周（P0） +- **Phase 2**：2-3 周（P0） +- **Phase 3**：2-3 周（P1） +- **Phase 4**：1-2 周（P2） +- **Phase 5**：1-2 周（P3） + +**总计**：8-13 周（约 2-3 个月） + +### 关键路径 + +``` +Phase 1 → Phase 2 → Phase 3 → Phase 4 → Phase 5 +``` + +**最短时间**：8 周（如果所有阶段都按最短时间完成） + +### 并行开发可能性 + +- **Phase 1 和 Phase 2**：可以部分并行（Phase 2 的 OpLogApplier 可以在 Phase 1 完成后开始） +- **Phase 3 和 Phase 4**：可以部分并行（快照集成和时序保证相对独立） +- **Phase 5**：可以在 Phase 1-4 完成后开始 + +## 优先级说明 + +### P0（最高优先级） + +- **Phase 1**：基础框架，所有后续功能都依赖于此 +- **Phase 2**：Standby 服务集成，核心功能 + +**必须完成**：这两个阶段是核心功能，必须优先完成。 + +### P1（高优先级） + +- **Phase 3**：时序保证和容错，确保数据一致性 + +**重要**：这个阶段确保数据一致性，应该在 Phase 1-2 完成后尽快完成。 + +### P2（中优先级） + +- **Phase 4**：快照集成和清理，减少存储压力 + +**可选但推荐**：这个阶段可以减少 etcd 存储压力，建议完成。 + +### P3（低优先级） + +- **Phase 5**：优化和完善，提升系统稳定性 + +**可选**：这个阶段是优化，可以在系统稳定运行后再完成。 + +## 实施建议 + +### 第一步：完成 Phase 1 + +1. **先实现 `EtcdOpLogStore` 的基础功能**（写入和读取） + - 确保可以成功写入和读取 OpLog + - 完成单元测试 + +2. **集成到 `OpLogManager`** + - 确保 Primary 可以写入 OpLog + - 完成集成测试 + +3. **扩展 `EtcdHelper`** + - 支持所有需要的操作 + - 完成单元测试 + +4. **完成测试** + - 单元测试、集成测试、性能测试 + +### 第二步：完成 Phase 2 + +1. **实现 `OpLogWatcher`** + - 支持 Watch etcd + - 完成单元测试 + +2. **实现 `OpLogApplier` 基础功能** + - 可以应用 OpLog + - 完成单元测试 + +3. **修改 `HotStandbyService`** + - 使用 etcd Watch + - 完成集成测试 + +4. **修改 `MasterServiceSupervisor`** + - 支持 Standby 模式 + - 完成端到端测试 + +### 第三步：完成 Phase 3 + +1. **实现序列号不连续处理** + - 缓存待处理条目 + - 从 etcd 读取缺失条目 + +2. **实现回滚和重放机制** + - 检测乱序 + - 回滚和重放 + +3. **完善错误处理和恢复** + - Watch 重连 + - 错误重试 + +4. **完成压力测试** + +### 第四步：完成 Phase 4 和 Phase 5 + +1. **实现快照集成和 OpLog 清理** + - 快照时记录 sequence_id + - 清理旧的 OpLog + +2. **实现 Standby 提升时的 Lease 初始化** + - 初始化所有对象的 lease + - 执行驱逐清理 + +3. **优化性能和完善监控** + - 批量写入（可选） + - OpLog 压缩（可选） + - 监控和告警 + +4. **完成所有测试** + +## 关键成功因素 + +### 1. 代码质量 + +- **代码审查**：每个阶段完成后进行代码审查 +- **单元测试覆盖率**：目标 > 80% +- **集成测试**：确保各组件正确集成 + +### 2. 性能要求 + +- **OpLog 写入性能**：> 1000 ops/s +- **Standby 同步延迟**：< 100ms +- **etcd 存储大小**：10 分钟内 < 1GB + +### 3. 稳定性要求 + +- **错误处理**：所有错误都有完善的处理 +- **自动恢复**：Watch 断开、读取失败等可以自动恢复 +- **监控告警**：关键指标都有监控和告警 + +### 4. 文档要求 + +- **设计文档**：所有设计都有详细文档 +- **API 文档**：所有接口都有文档 +- **运维文档**：部署和运维指南 + +## 总结 + +本实施计划按照依赖关系和重要性，将整个项目分为 5 个阶段。**Phase 1 和 Phase 2 是核心功能，必须优先完成**。Phase 3 确保数据一致性，Phase 4 和 Phase 5 是优化和完善。 + +**建议按照阶段顺序实施，每个阶段完成后进行充分测试，确保稳定性后再进入下一阶段。** + +### 关键要点 + +1. **优先级明确**：P0 > P1 > P2 > P3 +2. **依赖关系清晰**：Phase 1 → Phase 2 → Phase 3 → Phase 4 → Phase 5 +3. **测试充分**：每个阶段都有对应的测试要求 +4. **风险可控**：识别了高风险项并提供了缓解措施 +5. **时间合理**：总计 8-13 周，符合项目时间要求 + +### 下一步行动 + +1. **评审本计划**：与团队评审实施计划 +2. **分配任务**：根据计划分配开发任务 +3. **开始 Phase 1**：从基础框架开始实施 +4. **定期检查**：每周检查进度，确保按计划进行 + diff --git a/doc/zh/rfc-oplog-key-sequence-map-cleanup.md b/doc/zh/rfc-oplog-key-sequence-map-cleanup.md new file mode 100644 index 0000000000..6d954d46dc --- /dev/null +++ b/doc/zh/rfc-oplog-key-sequence-map-cleanup.md @@ -0,0 +1,253 @@ +# OpLogApplier key_sequence_map_ 清理策略 + +## 问题描述 + +在 Standby 端的 `OpLogApplier` 中，`key_sequence_map_` 用于跟踪每个 key 的 `key_sequence_id`，以确保 OpLog 的顺序正确性。当 metadata 被删除（REMOVE 操作）后，`key_sequence_map_` 中的条目仍然保留，用于检测可能的乱序操作。 + +### 内存泄漏风险 + +如果 `key_sequence_map_` 中的条目一直不删除，长期运行可能导致内存泄漏： + +- **内存占用**：每个条目约 90 字节（string key + uint64_t value + hash map 开销） +- **累积效应**：系统长期运行，可能有数百万个不同的 key 曾经存在过 +- **极端场景**：如果每天创建 10 万个新 key，运行 100 天，累计 1000 万个不同的 key，内存占用可达 900MB + +### 清理需求 + +需要在保证功能正确性的前提下，实现内存清理机制。 + +## 解决方案 + +### 核心策略 + +**清理条件**： +1. 最后一次操作是 `REMOVE`（DELETE） +2. 距离当前时间超过 1 小时 + +**清理频率**：每小时扫描一次 + +**保留策略**： +- `PUT_END` 和 `PUT_REVOKE` 操作的 key 不清理（metadata 可能仍存在） +- 即使超过 1 小时，只要最后操作不是 `REMOVE`，也保留 + +### 设计原理 + +1. **乱序检测时间窗口**：乱序检测一般只需要秒级的时间窗口，1 小时的保留时间足够处理网络延迟、重传等情况 +2. **只清理 DELETE 操作**：因为 DELETE 操作的 metadata 已经不存在，且超过 1 小时后不太可能再出现乱序 +3. **保留 PUT 操作**：PUT 操作的 metadata 可能仍存在，需要保留用于顺序检查 + +## 实现设计 + +### 数据结构 + +```cpp +class OpLogApplier { +private: + struct KeySequenceInfo { + uint64_t sequence_id{0}; + OpType last_op_type{OpType::PUT_END}; + std::chrono::steady_clock::time_point last_op_time; + + KeySequenceInfo() + : last_op_time(std::chrono::steady_clock::now()) {} + }; + + std::unordered_map key_sequence_map_; + mutable std::mutex key_sequence_mutex_; + + // 清理配置 + static constexpr std::chrono::hours kCleanupInterval{1}; // 每小时清理一次 + static constexpr std::chrono::hours kStaleThreshold{1}; // 1小时未访问则清理（仅限DELETE） + + std::chrono::steady_clock::time_point last_cleanup_time_; +}; +``` + +### 核心方法 + +#### 1. 定期清理检查 + +```cpp +void OpLogApplier::PeriodicCleanup() { + auto now = std::chrono::steady_clock::now(); + if (now - last_cleanup_time_ < kCleanupInterval) { + return; // 还没到清理时间 + } + + CleanupStaleKeySequences(); + last_cleanup_time_ = now; +} +``` + +#### 2. 清理过期条目 + +```cpp +void OpLogApplier::CleanupStaleKeySequences() { + std::lock_guard lock(key_sequence_mutex_); + auto now = std::chrono::steady_clock::now(); + auto threshold = now - kStaleThreshold; + + size_t cleaned = 0; + for (auto it = key_sequence_map_.begin(); + it != key_sequence_map_.end();) { + const auto& info = it->second; + + // 清理条件： + // 1. 最后一次操作是 REMOVE（DELETE） + // 2. 且距离当前超过1小时 + if (info.last_op_type == OpType::REMOVE && + info.last_op_time < threshold) { + it = key_sequence_map_.erase(it); + cleaned++; + } else { + ++it; + } + } + + if (cleaned > 0) { + LOG(INFO) << "Cleaned up " << cleaned + << " stale key_sequence_map entries " + << "(REMOVE operations older than 1 hour)"; + } +} +``` + +#### 3. 应用 OpLog 时更新 + +```cpp +bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { + // 1. 检查顺序 + if (!CheckSequenceOrder(entry)) { + // 处理乱序... + return false; + } + + // 2. 应用操作 + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + } + + // 3. 更新 key_sequence_map_ + { + std::lock_guard lock(key_sequence_mutex_); + auto& info = key_sequence_map_[entry.object_key]; + info.sequence_id = entry.key_sequence_id; + info.last_op_type = entry.op_type; + info.last_op_time = std::chrono::steady_clock::now(); + } + + // 4. 定期清理（每次应用时检查，避免额外线程） + PeriodicCleanup(); + + return true; +} +``` + +## 关键设计要点 + +### 1. 清理时机 + +- **触发方式**：在 `ApplyOpLogEntry` 中检查，无需额外线程 +- **清理频率**：每小时执行一次 +- **清理条件**：只清理 `REMOVE` 操作且超过 1 小时的条目 + +### 2. 安全性保证 + +- **保留 PUT 操作**：`PUT_END` 和 `PUT_REVOKE` 的 key 不清理，因为 metadata 可能仍存在 +- **1 小时窗口**：足够处理网络延迟、重传等异常情况 +- **线程安全**：使用 mutex 保护 `key_sequence_map_` 的访问 + +### 3. 内存占用控制 + +**清理前**： +- 假设系统长期运行，有 100 万个不同的 key 曾经存在过 +- 内存占用：100万 × 90字节 ≈ 90MB + +**清理后**： +- 假设系统每小时处理 10 万个 OpLog，其中 10% 是 REMOVE 操作 +- `key_sequence_map_` 中最多保留： + - 最近 1 小时的 REMOVE key：约 1 万个 + - 所有 PUT_END/PUT_REVOKE 的 key：取决于实际 metadata 数量 +- 内存占用：约 `(活跃key数量 + 1万) × 90字节` + +**内存节省**：从 90MB 降低到约 `(活跃key数量 + 1万) × 90字节`，通常远小于不清理的情况。 + +## 使用场景示例 + +### 场景 1：正常 REMOVE 操作 + +``` +时间线： +1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5, op_type=PUT_END + → 应用成功，key_sequence_map_["obj1"] = {seq:5, op:PUT_END, time:10:00} + +2. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=6, op_type=REMOVE + → 应用成功，key_sequence_map_["obj1"] = {seq:6, op:REMOVE, time:10:05} + → metadata 被删除 + +3. 1小时后（11:05），清理扫描 + → 检测到 "obj1" 的 last_op_type=REMOVE 且超过1小时 + → 清理 key_sequence_map_["obj1"] +``` + +### 场景 2：乱序 REMOVE 操作 + +``` +时间线： +1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5, op_type=PUT_END + → 应用成功，key_sequence_map_["obj1"] = {seq:5, op:PUT_END, time:10:00} + +2. Standby 收到 OpLog: sequence_id=102, key="obj1", key_sequence_id=6, op_type=PUT_END + → 应用成功，key_sequence_map_["obj1"] = {seq:6, op:PUT_END, time:10:02} + +3. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=5, op_type=REMOVE + → 检测到乱序：entry.key_sequence_id(5) <= current(6) + → 触发回滚和重放 + → key_sequence_map_["obj1"] 保留用于重放 +``` + +### 场景 3：删除后重新创建 + +``` +时间线： +1. key="obj1" 被 REMOVE，key_sequence_map_["obj1"] = {seq:6, op:REMOVE, time:10:00} + +2. 30分钟后（10:30），Standby 收到 OpLog: sequence_id=200, key="obj1", key_sequence_id=7, op_type=PUT_END + → 检查：key_sequence_map_["obj1"] 存在，seq=6（期望） + → 应用成功，key_sequence_map_["obj1"] = {seq:7, op:PUT_END, time:10:30} + → 不会被清理（因为 last_op_type=PUT_END） +``` + +## 配置参数 + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `kCleanupInterval` | 1 小时 | 清理检查的间隔时间 | +| `kStaleThreshold` | 1 小时 | REMOVE 操作超过此时间后可以清理 | + +## 优势 + +1. **内存控制**：及时清理已删除且超过 1 小时的 key，有效控制内存占用 +2. **安全性**：保留最近删除的 key，确保乱序检测的正确性 +3. **简单高效**：无需额外线程，在应用 OpLog 时检查，实现简单 +4. **精确清理**：只清理符合条件的条目，不影响活跃 key + +## 注意事项 + +1. **清理时机**：清理在 `ApplyOpLogEntry` 中触发，如果长时间没有 OpLog，可能不会及时清理 +2. **时间精度**：使用 `std::chrono::steady_clock`，不受系统时间调整影响 +3. **线程安全**：所有对 `key_sequence_map_` 的访问都需要加锁保护 + +## 相关文档 + +- [OpLog 主备同步完整方案](./rfc-oplog-via-etcd-complete-design.md) +- [OpLog 序列号乱序时的回滚和重放方案](./rfc-oplog-rollback-replay-on-sequence-violation.md) + diff --git a/doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md b/doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md new file mode 100644 index 0000000000..5b3338521f --- /dev/null +++ b/doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md @@ -0,0 +1,653 @@ +# OpLog 序列号乱序时的回滚和重放方案 + +## 问题描述 + +当 Standby 检测到某个 key 的 `key_sequence_id` 乱序时（例如：收到了 `key_sequence_id=5`，但之前已经处理了 `key_sequence_id=6`），说明该 key 的 metadata 可能已经不一致。 + +### 问题场景 + +``` +时间线： +1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5, op_type=PUT_END +2. Standby 应用成功，metadata 中 obj1 的 key_sequence_id = 5 +3. Standby 收到 OpLog: sequence_id=102, key="obj1", key_sequence_id=6, op_type=PUT_END +4. Standby 应用成功，metadata 中 obj1 的 key_sequence_id = 6 +5. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=5, op_type=REMOVE + ❌ 乱序！key_sequence_id=5 < 当前值 6 +``` + +**问题**： +- 该 key 的 metadata 可能已经不一致 +- 需要修复该 key 的数据状态 + +## 解决方案 + +### 核心思路 + +**对于乱序的 key，执行回滚和重放**： +1. **回滚**：从 metadata_store 中删除该 key 的所有状态 +2. **重放**：从该 key 第一次出现的 sequence_id 开始，从 etcd 重新读取所有 OpLog +3. **重写**：按正确顺序重新应用所有 OpLog，重建 metadata + +### 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ OpLogApplier::ApplyOpLogEntry() │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 检查 key_sequence_id 是否递增 │ +│ - 如果乱序 → 触发回滚和重放 │ +└─────────────────────────────────────────────────────────┘ + │ + ├─ 正常顺序 + │ │ + │ ▼ + │ ┌─────────────────────────────────────────┐ + │ │ 正常应用 OpLog │ + │ └─────────────────────────────────────────┘ + │ + └─ 乱序 + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 1. 回滚：删除该 key 的 metadata │ +│ - metadata_store_->RemoveKey(key) │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 2. 从 etcd 重新读取该 key 的所有 OpLog │ +│ - ReadOpLogForKey(key, first_seq_id) │ +│ - 过滤出该 key 的条目 │ +│ - 按 sequence_id 排序 │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 3. 按顺序重新应用所有 OpLog │ +│ - 跳过时序检查（因为已经排序） │ +│ - 重新构建 metadata │ +└─────────────────────────────────────────────────────────┘ +``` + +## 实现设计 + +### 1. 方案 A：基于 etcd 的完整重放（推荐） + +**优点**： +- 数据准确：从 etcd 读取保证数据正确 +- 实现简单：不需要维护操作历史 +- 内存友好：不需要额外存储 +- 容错性好：即使本地状态丢失也能恢复 + +**缺点**： +- 需要从 etcd 读取：可能有网络 I/O 开销 +- 可能较慢：如果该 key 的操作很多 + +#### 实现代码 + +```cpp +class OpLogApplier { +private: + // 记录每个 key 的首次 sequence_id（用于回滚） + std::unordered_map key_first_sequence_id_; + std::mutex key_first_sequence_mutex_; + + // 记录正在回滚的 key（防止并发回滚） + std::set keys_under_rollback_; + std::mutex rollback_mutex_; + + EtcdOpLogStore* etcd_oplog_store_; + +public: + bool ApplyOpLogEntry(const OpLogEntry& entry) { + // 1. 记录首次 sequence_id + { + std::lock_guard lock(key_first_sequence_mutex_); + if (key_first_sequence_id_.count(entry.object_key) == 0) { + key_first_sequence_id_[entry.object_key] = entry.sequence_id; + } + } + + // 2. 检查 key 级别的时序性 + if (!CheckSequenceOrder(entry)) { + LOG(ERROR) << "Key-level sequence order violation for key: " + << entry.object_key + << ", entry_seq=" << entry.key_sequence_id + << ", current_seq=" << GetKeySequenceId(entry.object_key); + + // 3. 触发回滚和重放（异步执行，不阻塞） + std::thread([this, key = entry.object_key]() { + RollbackAndReplayKey(key); + }).detach(); + + // 暂时跳过这个条目，等待回滚完成 + return false; + } + + // 4. 正常应用 + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + } + + // 5. 更新 key_sequence_map_ + { + std::lock_guard lock(key_sequence_mutex_); + key_sequence_map_[entry.object_key] = entry.key_sequence_id; + } + + return true; + } + +private: + bool RollbackAndReplayKey(const std::string& key) { + // 1. 检查是否正在回滚（防止并发回滚） + { + std::lock_guard lock(rollback_mutex_); + if (keys_under_rollback_.count(key) > 0) { + LOG(WARNING) << "Key is already under rollback: " << key; + return false; + } + keys_under_rollback_.insert(key); + } + + // 2. 获取该 key 的首次 sequence_id + uint64_t first_seq_id; + { + std::lock_guard lock(key_first_sequence_mutex_); + auto it = key_first_sequence_id_.find(key); + if (it == key_first_sequence_id_.end()) { + LOG(ERROR) << "Cannot find first sequence_id for key: " << key; + std::lock_guard lock2(rollback_mutex_); + keys_under_rollback_.erase(key); + return false; + } + first_seq_id = it->second; + } + + // 3. 回滚：从 metadata_store_ 中删除该 key + LOG(INFO) << "Rolling back key: " << key + << ", removing from metadata_store"; + metadata_store_->RemoveKey(key); + + // 4. 从 etcd 重新读取该 key 的所有 OpLog + LOG(INFO) << "Re-reading OpLog for key: " << key + << " from sequence_id: " << first_seq_id; + + std::vector key_entries; + if (!ReadOpLogForKey(key, first_seq_id, key_entries)) { + LOG(ERROR) << "Failed to read OpLog for key: " << key; + std::lock_guard lock(rollback_mutex_); + keys_under_rollback_.erase(key); + return false; + } + + // 5. 按顺序重新应用所有 OpLog + LOG(INFO) << "Replaying " << key_entries.size() + << " OpLog entries for key: " << key; + + for (const auto& entry : key_entries) { + // 重新应用（跳过时序检查，因为我们已经从 etcd 读取了正确的顺序） + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + } + + // 更新 key_sequence_map_ + { + std::lock_guard lock(key_sequence_mutex_); + key_sequence_map_[key] = entry.key_sequence_id; + } + } + + // 6. 清除回滚标记 + { + std::lock_guard lock(rollback_mutex_); + keys_under_rollback_.erase(key); + } + + LOG(INFO) << "Successfully replayed OpLog for key: " << key; + return true; + } + + bool ReadOpLogForKey(const std::string& key, + uint64_t start_seq_id, + std::vector& entries) { + // 从 etcd 读取从 start_seq_id 开始的所有 OpLog + std::vector all_entries; + const uint32_t batch_size = 10000; // 批量读取 + + uint64_t current_seq_id = start_seq_id; + while (true) { + std::vector batch; + if (!etcd_oplog_store_->ReadOpLogSince(current_seq_id, batch_size, batch)) { + LOG(ERROR) << "Failed to read OpLog from etcd"; + return false; + } + + if (batch.empty()) { + break; // 没有更多条目 + } + + // 过滤出该 key 的条目 + for (const auto& entry : batch) { + if (entry.object_key == key) { + entries.push_back(entry); + } + } + + // 更新 current_seq_id + if (batch.size() < batch_size) { + break; // 已读取完所有条目 + } + current_seq_id = batch.back().sequence_id + 1; + } + + // 按 sequence_id 排序（确保顺序正确） + std::sort(entries.begin(), entries.end(), + [](const OpLogEntry& a, const OpLogEntry& b) { + return a.sequence_id < b.sequence_id; + }); + + return true; + } +}; +``` + +### 2. 方案 B：基于操作历史的回滚（可选） + +**优点**： +- 快速：不需要网络 I/O +- 高效：直接从内存读取 + +**缺点**： +- 需要额外内存：存储操作历史 +- 实现复杂：需要维护历史记录 +- 容错性差：如果历史丢失，无法恢复 + +#### 实现代码 + +```cpp +class OpLogApplier { +private: + // 记录每个 key 的操作历史（用于回滚） + struct KeyOperationHistory { + std::vector operations; // 按顺序记录的操作 + uint64_t first_sequence_id{0}; + }; + std::unordered_map key_history_; + std::mutex key_history_mutex_; + + // 限制历史记录的大小（避免内存无限增长） + static constexpr size_t kMaxHistorySize = 1000; + +public: + bool ApplyOpLogEntry(const OpLogEntry& entry) { + // 1. 记录操作历史 + { + std::lock_guard lock(key_history_mutex_); + auto& history = key_history_[entry.object_key]; + if (history.operations.empty()) { + history.first_sequence_id = entry.sequence_id; + } + + // 限制历史记录大小 + if (history.operations.size() < kMaxHistorySize) { + history.operations.push_back(entry); + } else { + // 如果超过限制，只保留最近的操作 + history.operations.erase(history.operations.begin()); + history.operations.push_back(entry); + } + } + + // 2. 检查时序性 + if (!CheckSequenceOrder(entry)) { + return RollbackAndReplayKey(entry.object_key); + } + + // 3. 正常应用 + // ... + } + +private: + bool RollbackAndReplayKey(const std::string& key) { + std::lock_guard lock(key_history_mutex_); + + auto it = key_history_.find(key); + if (it == key_history_.end()) { + LOG(ERROR) << "Cannot find history for key: " << key; + return false; + } + + // 1. 回滚：删除该 key 的 metadata + metadata_store_->RemoveKey(key); + + // 2. 重新应用所有操作（从历史记录中） + for (const auto& entry : it->second.operations) { + // 重新应用 + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + } + + // 更新 key_sequence_map_ + { + std::lock_guard lock2(key_sequence_mutex_); + key_sequence_map_[key] = entry.key_sequence_id; + } + } + + return true; + } +}; +``` + +## 关键设计点 + +### 1. 回滚起点的确定 + +**方案 A（推荐）**： +- 维护 `key_first_sequence_id_` 记录每个 key 第一次出现的 sequence_id +- 从该 sequence_id 开始重新读取所有 OpLog + +**方案 B**： +- 维护操作历史，从历史记录中获取所有操作 + +### 2. 并发处理 + +**问题**：回滚期间，如果收到新的 OpLog 怎么办？ + +**解决方案**： +- 使用 `keys_under_rollback_` 标记正在回滚的 key +- 回滚期间，新的 OpLog 暂时跳过（返回 false） +- 回滚完成后，新的 OpLog 可以正常处理 + +```cpp +bool ApplyOpLogEntry(const OpLogEntry& entry) { + // 检查是否正在回滚 + { + std::lock_guard lock(rollback_mutex_); + if (keys_under_rollback_.count(entry.object_key) > 0) { + LOG(WARNING) << "Key is under rollback, skipping entry: " + << entry.sequence_id; + return false; // 暂时跳过，等待回滚完成 + } + } + + // 正常处理 + // ... +} +``` + +### 3. 性能优化 + +#### 3.1 异步回滚 + +**问题**：回滚和重放可能耗时，会阻塞新 OpLog 的处理 + +**解决方案**：异步执行回滚，不阻塞正常处理 + +```cpp +if (!CheckSequenceOrder(entry)) { + // 异步回滚（不阻塞） + std::thread([this, key = entry.object_key]() { + RollbackAndReplayKey(key); + }).detach(); + + return false; // 暂时跳过 +} +``` + +#### 3.2 批量读取 + +**问题**：从 etcd 读取大量 OpLog 可能较慢 + +**解决方案**：批量读取，减少网络往返 + +```cpp +bool ReadOpLogForKey(const std::string& key, + uint64_t start_seq_id, + std::vector& entries) { + const uint32_t batch_size = 10000; // 批量读取 + uint64_t current_seq_id = start_seq_id; + + while (true) { + std::vector batch; + etcd_oplog_store_->ReadOpLogSince(current_seq_id, batch_size, batch); + // ... + } +} +``` + +#### 3.3 限制回滚范围 + +**问题**：如果该 key 的操作非常多，回滚可能很耗时 + +**解决方案**：限制回滚范围，只回滚最近的操作 + +```cpp +bool RollbackAndReplayKey(const std::string& key) { + // 只回滚最近 N 个操作 + const uint64_t max_rollback_ops = 1000; + + // 从 etcd 读取时，限制范围 + uint64_t start_seq_id = std::max( + first_seq_id, + GetLatestSequenceId() - max_rollback_ops + ); + + // ... +} +``` + +### 4. 错误处理 + +#### 4.1 回滚失败 + +**场景**：从 etcd 读取 OpLog 失败 + +**处理**： +- 记录错误日志 +- 清除回滚标记 +- 可以考虑触发全量同步 + +```cpp +if (!ReadOpLogForKey(key, first_seq_id, key_entries)) { + LOG(ERROR) << "Failed to read OpLog for key: " << key; + + // 清除回滚标记 + { + std::lock_guard lock(rollback_mutex_); + keys_under_rollback_.erase(key); + } + + // 可选：触发全量同步 + // TriggerFullSync(); + + return false; +} +``` + +#### 4.2 重复回滚 + +**场景**：同一个 key 多次触发回滚 + +**处理**： +- 使用 `keys_under_rollback_` 防止并发回滚 +- 如果正在回滚，跳过新的回滚请求 + +### 5. 监控和告警 + +#### 5.1 记录乱序频率 + +```cpp +class OpLogApplier { +private: + // 记录每个 key 的乱序次数 + std::unordered_map key_violation_count_; + std::mutex violation_count_mutex_; + + // 乱序阈值 + static constexpr uint64_t kMaxViolationsPerKey = 10; + +public: + bool ApplyOpLogEntry(const OpLogEntry& entry) { + if (!CheckSequenceOrder(entry)) { + // 记录乱序次数 + { + std::lock_guard lock(violation_count_mutex_); + key_violation_count_[entry.object_key]++; + + if (key_violation_count_[entry.object_key] > kMaxViolationsPerKey) { + LOG(ERROR) << "Too many violations for key: " + << entry.object_key + << ", count: " + << key_violation_count_[entry.object_key]; + + // 触发全量同步 + TriggerFullSync(); + return false; + } + } + + // 触发回滚 + // ... + } + } +}; +``` + +#### 5.2 性能指标 + +- 回滚次数 +- 回滚耗时 +- 回滚成功率 +- 乱序频率 + +## 方案对比 + +| 特性 | 方案 A（基于 etcd） | 方案 B（基于历史） | +|------|-------------------|------------------| +| **数据准确性** | 高（从 etcd 读取） | 中（依赖历史记录） | +| **实现复杂度** | 低 | 高 | +| **内存开销** | 低 | 高（需要存储历史） | +| **性能** | 中（需要网络 I/O） | 高（内存操作） | +| **容错性** | 高（可以从 etcd 恢复） | 低（历史可能丢失） | +| **适用场景** | 乱序不频繁、数据准确性要求高 | 乱序频繁、性能要求高 | + +## 推荐方案 + +**推荐使用方案 A（基于 etcd 的完整重放）**，原因： + +1. **数据准确性高**：从 etcd 读取保证数据正确 +2. **实现简单**：不需要维护操作历史 +3. **内存友好**：不需要额外存储 +4. **容错性好**：即使本地状态丢失也能恢复 + +**优化建议**： +1. **异步回滚**：不阻塞正常处理 +2. **批量读取**：减少网络往返 +3. **限制范围**：只回滚最近的操作 +4. **监控告警**：记录乱序频率，超过阈值时触发全量同步 + +## 测试场景 + +### 1. 正常乱序检测和回滚 + +``` +1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5 +2. 应用成功 +3. Standby 收到 OpLog: sequence_id=102, key="obj1", key_sequence_id=6 +4. 应用成功 +5. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=5 +6. 检测到乱序，触发回滚 +7. 从 etcd 重新读取 obj1 的所有 OpLog +8. 按顺序重新应用 +9. 验证 metadata 正确 +``` + +### 2. 并发回滚保护 + +``` +1. 检测到 key="obj1" 乱序，开始回滚 +2. 回滚期间，收到新的 OpLog: key="obj1" +3. 检测到正在回滚，跳过新 OpLog +4. 回滚完成后，新的 OpLog 可以正常处理 +``` + +### 3. 回滚失败处理 + +``` +1. 检测到乱序，触发回滚 +2. 从 etcd 读取 OpLog 失败 +3. 记录错误日志 +4. 清除回滚标记 +5. 可选：触发全量同步 +``` + +### 4. 频繁乱序处理 + +``` +1. 某个 key 频繁乱序（超过阈值） +2. 记录告警 +3. 触发全量同步 +4. 避免频繁回滚影响性能 +``` + +## 总结 + +### 核心方案 + +**对于乱序的 key，执行回滚和重放**： +1. 回滚：删除该 key 的 metadata +2. 重放：从 etcd 重新读取该 key 的所有 OpLog +3. 重写：按正确顺序重新应用所有 OpLog + +### 关键实现 + +1. **回滚起点**：维护 `key_first_sequence_id_` 记录首次 sequence_id +2. **并发保护**：使用 `keys_under_rollback_` 防止并发回滚 +3. **异步执行**：回滚在后台线程执行，不阻塞正常处理 +4. **批量读取**：从 etcd 批量读取 OpLog，减少网络往返 +5. **监控告警**：记录乱序频率，超过阈值时触发全量同步 + +### 优势 + +1. **数据准确性**：从 etcd 读取保证数据正确 +2. **局部修复**：只影响乱序的 key，不影响其他 key +3. **自动恢复**：自动检测和修复数据不一致 +4. **性能友好**：异步执行，不阻塞正常处理 + +### 注意事项 + +1. **性能影响**：回滚和重放可能耗时，需要异步执行 +2. **并发处理**：回滚期间需要防止并发处理该 key +3. **范围限制**：可以限制回滚范围，只回滚最近的操作 +4. **监控告警**：需要监控乱序频率，超过阈值时考虑全量同步 + diff --git a/doc/zh/rfc-oplog-via-etcd-complete-design.md b/doc/zh/rfc-oplog-via-etcd-complete-design.md new file mode 100644 index 0000000000..58060f3098 --- /dev/null +++ b/doc/zh/rfc-oplog-via-etcd-complete-design.md @@ -0,0 +1,738 @@ +# 基于 etcd 的 OpLog 主备同步完整方案 + +## 方案概述 + +使用 etcd 作为中间可靠性组件，实现 Primary Master 和 Standby Master 之间的 OpLog 同步。OpLog 只记录 PUT 和 DELETE 事件，通过 etcd 的 Watch 机制实现实时同步。 + +## 核心设计原则 + +1. **OpLog 只记录 PUT 和 DELETE 事件**：不记录 LEASE_RENEW，减少 OpLog 大小 +2. **etcd 作为中间存储**：利用 etcd 的强一致性和 Watch 机制 +3. **快照集成**：与现有快照机制集成，快照后可以清理旧的 OpLog +4. **时序保证**：通过 sequence_id 和 key 级别的版本控制保证时序 + +## 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ Primary Master │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ MasterService│ │ OpLogManager │ │ +│ │ │ │ │ │ +│ │ PutEnd() │─────▶│ Append() │ │ +│ │ Remove() │ │ │ │ +│ │ Eviction │ └──────────────┘ │ +│ └──────────────┘ │ │ +│ │ │ +│ ▼ │ +│ ┌──────────────┐ │ +│ │ EtcdOpLogStore│ │ +│ │ │ │ +│ │ WriteOpLog() │ │ +│ └──────────────┘ │ +│ │ │ +│ │ 写入 etcd │ +│ ▼ │ +│ ┌──────────────┐ │ +│ │ etcd │ │ +│ │ │ │ +│ │ /oplog/{seq} │ │ +│ └──────────────┘ │ +└─────────────────────────────────────────────────────────┘ + │ + │ Watch + ▼ +┌─────────────────────────────────────────────────────────┐ +│ Standby Masters │ +│ │ +│ ┌──────────────────────────────────────────────────┐ │ +│ │ MasterServiceSupervisor │ │ +│ │ - 检测 leader │ │ +│ │ - 启动/停止 HotStandbyService │ │ +│ └──────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ OpLogWatcher │ │ OpLogApplier │ │ +│ │ │ │ │ │ +│ │ WatchEtcd() │─────▶│ ApplyOpLog() │ │ +│ │ │ │ │ │ +│ └──────────────┘ └──────────────┘ │ +│ │ │ │ +│ │ ▼ │ +│ │ ┌──────────────┐ │ +│ │ │ MetadataStore│ │ +│ │ │ │ │ +│ │ │ 更新 metadata │ │ +│ │ └──────────────┘ │ +│ │ │ +│ └──────────────────────────────────────────────┘ +└─────────────────────────────────────────────────────────┘ +``` + +## etcd Key 设计 + +### 1. OpLog Entry Key + +``` +mooncake-store/oplog/{cluster_id}/{sequence_id} +``` + +**示例**： +``` +mooncake-store/oplog/mooncake_cluster/1 +mooncake-store/oplog/mooncake_cluster/2 +mooncake-store/oplog/mooncake_cluster/3 +... +``` + +**设计考虑**： +- 使用 `sequence_id` 作为 key 的一部分，保证顺序 +- 支持按 sequence_id 范围查询 +- 易于清理（删除指定 sequence_id 之前的 key） + +### 2. 最新 Sequence ID Key + +``` +mooncake-store/oplog/{cluster_id}/latest +``` + +**用途**： +- 存储当前最新的 sequence_id +- Standby 可以快速获取最新的 sequence_id +- 用于快照时记录 OpLog 的 sequence_id + +### 3. 快照 Sequence ID Key + +``` +mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id +``` + +**用途**： +- 记录每个快照对应的 sequence_id +- 用于确定可以清理的 OpLog 范围 + +## OpLog Entry 数据结构 + +```cpp +struct OpLogEntry { + uint64_t sequence_id{0}; // 全局递增序列号 + uint64_t timestamp_ms{0}; // 时间戳（毫秒） + OpType op_type{OpType::PUT_END}; // PUT_END, PUT_REVOKE, REMOVE + std::string object_key; // 对象 key + std::string payload; // 可选负载（用于 PUT_END 时携带 replica 信息） + uint32_t checksum{0}; // 校验和 + uint32_t prefix_hash{0}; // key 前缀哈希 + uint64_t key_sequence_id{0}; // 该 key 的操作序列号（用于时序保证） +}; +``` + +**JSON 序列化格式**： +```json +{ + "sequence_id": 12345, + "timestamp": 1704110400123, + "op_type": "PUT_END", + "key": "object_key_123", + "payload": "optional_payload", + "checksum": 1234567890, + "prefix_hash": 987654321, + "key_sequence_id": 5 +} +``` + +## Primary 端实现 + +### 1. EtcdOpLogStore 类 + +```cpp +class EtcdOpLogStore { +public: + EtcdOpLogStore(const std::string& etcd_endpoints, + const std::string& cluster_id); + + // 写入 OpLog 到 etcd + bool WriteOpLog(const OpLogEntry& entry); + + // 批量写入 OpLog（可选优化） + bool WriteOpLogBatch(const std::vector& entries); + + // 更新最新的 sequence_id + bool UpdateLatestSequenceId(uint64_t sequence_id); + + // 记录快照对应的 sequence_id + bool RecordSnapshotSequenceId(const std::string& snapshot_id, + uint64_t sequence_id); + + // 清理指定 sequence_id 之前的 OpLog + bool CleanupOpLogBefore(uint64_t sequence_id); + +private: + std::string BuildOpLogKey(uint64_t sequence_id); + std::string SerializeOpLogEntry(const OpLogEntry& entry); + OpLogEntry DeserializeOpLogEntry(const std::string& data); + + std::string etcd_prefix_; + std::string cluster_id_; + // etcd client +}; +``` + +### 2. 集成到 OpLogManager + +```cpp +class OpLogManager { +public: + // 设置 EtcdOpLogStore（可选，如果不设置则只写入内存） + void SetEtcdOpLogStore(EtcdOpLogStore* store); + + uint64_t Append(OpType type, const std::string& key, + const std::string& payload = std::string()) { + OpLogEntry entry; + // ... 填充 entry ... + + // 写入内存 buffer + buffer_.emplace_back(entry); + + // 写入 etcd（如果设置了） + if (etcd_store_) { + etcd_store_->WriteOpLog(entry); + etcd_store_->UpdateLatestSequenceId(entry.sequence_id); + } + + return entry.sequence_id; + } + +private: + EtcdOpLogStore* etcd_store_{nullptr}; + // ... 其他成员 ... +}; +``` + +### 3. 驱逐时记录 DELETE 事件 + +```cpp +void MasterService::BatchEvict(...) { + // ... 驱逐逻辑 ... + + if (it->second.lease_timeout <= target_timeout) { + std::string evicted_key = it->first; + + // 驱逐对象 + total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + it->second.EraseReplica(ReplicaType::MEMORY); + + if (it->second.IsValid() == false) { + // 对象完全无效，记录 DELETE 事件 + AppendOpLogAndNotify(OpType::REMOVE, evicted_key); + it = shard.metadata.erase(it); + } else { + ++it; + } + } +} +``` + +## Standby 端实现 + +### 0. Standby 服务集成 + +**问题**：现有代码中，Standby 在等待 leader 选举期间只是阻塞等待，没有运行 Standby 服务来同步 OpLog。 + +**解决方案**：在 Standby 模式下并行运行 Standby 服务，watch etcd OpLog 并实时恢复 metadata。 + +**核心流程**： +1. `MasterServiceSupervisor` 检测到有 leader 时，启动 `HotStandbyService` +2. `HotStandbyService` 启动 `OpLogWatcher` watch etcd OpLog +3. 实时应用 OpLog 到本地 metadata store +4. 选举成功后，停止 Standby 服务并提升为 Primary + +**详细设计请参考**：`doc/zh/rfc-standby-service-integration.md` + +### 1. OpLogWatcher 类 + +```cpp +class OpLogWatcher { +public: + OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, + OpLogApplier* applier); + + // 启动 Watch + void Start(); + + // 停止 Watch + void Stop(); + + // 从指定 sequence_id 开始读取历史 OpLog + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries); + +private: + // Watch etcd OpLog 变化 + void WatchOpLog(); + + // 处理 Watch 事件 + void HandleWatchEvent(const WatchEvent& event); + + std::string etcd_prefix_; + std::string cluster_id_; + OpLogApplier* applier_; + std::atomic running_{false}; + std::thread watch_thread_; + uint64_t last_processed_sequence_id_{0}; +}; +``` + +### 2. OpLogApplier 类（时序保证） + +```cpp +class OpLogApplier { +public: + OpLogApplier(MetadataStore* metadata_store); + + // 应用 OpLog Entry（带时序检查） + bool ApplyOpLogEntry(const OpLogEntry& entry); + + // 获取 key 的当前 sequence_id + uint64_t GetKeySequenceId(const std::string& key) const; + + // 恢复处理状态 + void Recover(uint64_t last_applied_sequence_id); + +private: + // 检查时序性 + bool CheckSequenceOrder(const OpLogEntry& entry); + + // 应用 PUT_END + void ApplyPutEnd(const OpLogEntry& entry); + + // 应用 PUT_REVOKE + void ApplyPutRevoke(const OpLogEntry& entry); + + // 应用 REMOVE + void ApplyRemove(const OpLogEntry& entry); + + MetadataStore* metadata_store_; + + // 记录每个 key 的最后 sequence_id（用于时序检查） + std::unordered_map key_sequence_map_; + std::mutex key_sequence_mutex_; + + // 记录待处理的条目（用于处理序列号不连续的情况） + std::map pending_entries_; + uint64_t expected_sequence_id_{1}; + std::mutex pending_mutex_; +}; +``` + +### 3. 时序保证机制 + +```cpp +bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { + // 1. 检查全局序列号连续性 + if (entry.sequence_id != expected_sequence_id_) { + if (entry.sequence_id > expected_sequence_id_) { + // 序列号不连续，缓存待处理 + std::lock_guard lock(pending_mutex_); + pending_entries_[entry.sequence_id] = entry; + + // 等待一段时间，看是否有缺失的条目到达 + ScheduleWaitForMissingEntries(entry.sequence_id); + return false; + } else { + // 序列号小于期望值（可能是重复或乱序） + LOG(WARNING) << "Received out-of-order OpLog entry: " + << "expected=" << expected_sequence_id_ + << ", received=" << entry.sequence_id; + return false; + } + } + + // 2. 检查 key 级别的时序性 + if (!CheckSequenceOrder(entry)) { + LOG(ERROR) << "Key-level sequence order violation for key: " + << entry.object_key + << ", entry_seq=" << entry.key_sequence_id + << ", current_seq=" << GetKeySequenceId(entry.object_key); + + // 触发回滚和重放（异步执行） + // 详细设计请参考：doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md + RollbackAndReplayKey(entry.object_key); + return false; + } + + // 3. 应用 OpLog + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + default: + LOG(WARNING) << "Unknown OpType: " + << static_cast(entry.op_type); + return false; + } + + // 4. 更新状态 + { + std::lock_guard lock(key_sequence_mutex_); + key_sequence_map_[entry.object_key] = entry.key_sequence_id; + } + + expected_sequence_id_++; + + // 5. 处理待处理的条目 + ProcessPendingEntries(); + + return true; +} + +bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { + std::lock_guard lock(key_sequence_mutex_); + + auto it = key_sequence_map_.find(entry.object_key); + if (it == key_sequence_map_.end()) { + // 新 key，允许 + return true; + } + + // 检查 key_sequence_id 是否递增 + if (entry.key_sequence_id <= it->second) { + // 序列号乱序，需要回滚和重放 + return false; + } + + return true; +} +``` + +### 4. 初始同步流程 + +```cpp +class StandbyInitialSync { +public: + void PerformInitialSync() { + // Step 1: 从 Primary 获取快照 + MetadataSnapshot snapshot = RequestSnapshotFromPrimary(); + + // Step 2: 获取快照对应的 sequence_id + uint64_t snapshot_seq_id = snapshot.last_oplog_sequence_id; + + // Step 3: 应用快照 + metadata_store_->ImportSnapshot(snapshot); + + // Step 4: 从 etcd 读取快照后的 OpLog + std::vector entries; + op_log_watcher_->ReadOpLogSince(snapshot_seq_id + 1, entries); + + // Step 5: 应用历史 OpLog + for (const auto& entry : entries) { + applier_->ApplyOpLogEntry(entry); + } + + // Step 6: 开始 Watch 增量 OpLog + op_log_watcher_->Start(); + } +}; +``` + +## 快照集成 + +### 1. 快照时记录 Sequence ID + +```cpp +class SnapshotManager { +public: + MetadataSnapshot CreateSnapshot() { + MetadataSnapshot snapshot; + + // 1. 导出 metadata + snapshot.metadata = ExportMetadata(); + + // 2. 记录当前的 OpLog sequence_id + snapshot.last_oplog_sequence_id = oplog_manager_->GetLastSequenceId(); + + // 3. 将快照信息写入 etcd + std::string snapshot_id = GenerateSnapshotId(); + etcd_oplog_store_->RecordSnapshotSequenceId( + snapshot_id, snapshot.last_oplog_sequence_id); + + // 4. 清理旧的 OpLog + etcd_oplog_store_->CleanupOpLogBefore( + snapshot.last_oplog_sequence_id); + + return snapshot; + } +}; +``` + +### 2. OpLog 清理策略 + +**方案：从 etcd 查询最小的 sequence_id，然后使用 DeleteRange 删除** + +```cpp +bool EtcdOpLogStore::CleanupOpLogBefore(uint64_t target_sequence_id) { + if (target_sequence_id <= 1) { + return true; // 没有需要清理的 + } + + // 1. 从 etcd 查询最小的 sequence_id + uint64_t min_seq_id = GetMinSequenceId(); + + // 2. 如果 min_seq_id >= target_sequence_id，无需清理 + if (min_seq_id >= target_sequence_id) { + return true; + } + + // 3. 执行 DeleteRange + std::string start_key = BuildOpLogKey(min_seq_id); + std::string end_key = BuildOpLogKey(target_sequence_id); + + int64_t deleted_count = 0; + auto err = EtcdHelper::DeleteRange( + start_key.c_str(), start_key.size(), + end_key.c_str(), end_key.size(), + deleted_count); + + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to cleanup OpLog"; + return false; + } + + LOG(INFO) << "Cleaned up " << deleted_count + << " OpLog entries from " << min_seq_id + << " to " << target_sequence_id; + return true; +} + +uint64_t EtcdOpLogStore::GetMinSequenceId() const { + // 从 etcd 查询最小的 OpLog sequence_id + std::string prefix = etcd_prefix_ + "/" + cluster_id_ + "/"; + std::string first_key, first_value; + + auto err = EtcdHelper::GetFirstKeyWithPrefix( + prefix.c_str(), prefix.size(), + first_key, first_value); + + if (err == ErrorCode::OK) { + // 从 key 中提取 sequence_id + uint64_t min_seq_id = ExtractSequenceIdFromKey(first_key); + if (min_seq_id > 0) { + return min_seq_id; + } + } + + // Fallback：从快照记录获取 + uint64_t last_snapshot_seq_id = GetLastSnapshotSequenceId(); + if (last_snapshot_seq_id > 0) { + return last_snapshot_seq_id; + } + + // 保守策略：从 1 开始 + return 1; +} +``` + +**详细实现请参考：`doc/zh/rfc-oplog-cleanup-start-sequence-id.md`** + +## 时序保证机制详解 + +### 1. 全局序列号（sequence_id） + +- **作用**：保证所有 OpLog 事件的全局顺序 +- **生成**：Primary 端 OpLogManager 全局递增 +- **检查**：Standby 端检查 sequence_id 是否连续 + +### 2. Key 级别序列号（key_sequence_id） + +- **作用**：保证同一个 key 的操作顺序 +- **生成**：Primary 端为每个 key 维护独立的序列号 +- **检查**：Standby 端检查 key_sequence_id 是否递增 + +### 3. 序列号不连续处理 + +```cpp +void OpLogApplier::ScheduleWaitForMissingEntries(uint64_t missing_seq) { + // 等待一段时间（如 1 秒） + std::this_thread::sleep_for(std::chrono::seconds(1)); + + // 如果缺失的条目仍未到达，需要从 etcd 读取 + if (pending_entries_.find(missing_seq) == pending_entries_.end()) { + RequestMissingOpLog(missing_seq); + } +} + +void OpLogApplier::RequestMissingOpLog(uint64_t missing_seq) { + // 从 etcd 读取缺失的 OpLog + OpLogEntry entry; + if (ReadOpLogFromEtcd(missing_seq, entry)) { + ApplyOpLogEntry(entry); + } else { + LOG(ERROR) << "Failed to read missing OpLog: seq=" << missing_seq; + // 触发重新同步 + TriggerResync(); + } +} +``` + +## 实现步骤 + +### Phase 1：基础框架（优先级：高） + +1. **实现 EtcdOpLogStore** + - 写入 OpLog 到 etcd + - 更新最新 sequence_id + - 读取 OpLog 从 etcd + +2. **集成到 OpLogManager** + - 添加 EtcdOpLogStore 成员 + - 在 Append 时写入 etcd + +3. **实现 OpLogWatcher** + - Watch etcd OpLog 变化 + - 处理 Watch 事件 + +### Phase 2：Standby 端处理（优先级：高） + +1. **实现 OpLogApplier** + - 应用 OpLog Entry + - 时序检查逻辑 + - 处理序列号不连续 + +2. **实现初始同步** + - 从 Primary 获取快照 + - 读取历史 OpLog + - 应用快照和 OpLog + +### Phase 3：快照集成（优先级：中） + +1. **快照时记录 sequence_id** + - 在快照中记录 last_oplog_sequence_id + - 写入 etcd + +2. **OpLog 清理** + - 实现 CleanupOpLogBefore + - 定期清理旧的 OpLog + +### Phase 4：优化（优先级：低） + +1. **批量写入** + - 实现 WriteOpLogBatch + - 减少 etcd 写入次数 + +2. **压缩** + - OpLog Entry 压缩 + - 减少 etcd 存储大小 + +## 关键设计要点 + +### 1. etcd Key 设计 + +- 使用顺序 Key：`mooncake-store/oplog/{cluster_id}/{sequence_id}` +- 支持按 sequence_id 范围查询 +- 易于清理（删除指定 sequence_id 之前的 key） + +### 2. 时序保证 + +- **全局序列号**：保证所有事件的全局顺序 +- **Key 级别序列号**：保证同一 key 的操作顺序 +- **序列号不连续处理**：检测并处理序列号不连续的情况 +- **序列号乱序处理**：检测到 key 级别乱序时，执行回滚和重放（详细设计请参考：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md`） + +### 3. 快照集成 + +- 快照时记录 sequence_id +- 快照后清理旧的 OpLog +- Standby 从快照点开始应用增量 OpLog + +### 4. 故障恢复 + +- Standby 持久化处理状态 +- 支持断点续传 +- 发现不一致时触发重新同步 + +### 5. Standby 服务集成 + +**问题**：现有代码中，Standby 在等待 leader 选举期间只是阻塞等待，没有运行 Standby 服务来同步 OpLog。 + +**解决方案**：在 Standby 模式下并行运行 Standby 服务，watch etcd OpLog 并实时恢复 metadata。 + +**详细设计请参考**：`doc/zh/rfc-standby-service-integration.md` + +### 6. Standby 提升为 Primary 时的 Lease 初始化 + +**问题**：Standby 上的对象 lease 都是 0（因为 OpLog 只包含 PUT_END，不包含续约信息），提升为 Primary 后所有对象会立即过期。 + +**解决方案**：在 `Promote()` 时，给所有 lease 为 0 的对象授予默认租约时间（`default_kv_lease_ttl`）。 + +**详细设计请参考**：`doc/zh/rfc-standby-promotion-lease-initialization.md` + +### 7. 序列号乱序时的回滚和重放 + +**问题**：当检测到某个 key 的 `key_sequence_id` 乱序时，该 key 的 metadata 可能已经不一致。 + +**解决方案**：对于乱序的 key，执行回滚和重放： +1. **回滚**：从 metadata_store 中删除该 key 的所有状态 +2. **重放**：从该 key 第一次出现的 sequence_id 开始，从 etcd 重新读取所有 OpLog +3. **重写**：按正确顺序重新应用所有 OpLog，重建 metadata + +**关键设计**： +- 异步执行回滚，不阻塞正常处理 +- 使用 `keys_under_rollback_` 防止并发回滚 +- 从 etcd 批量读取 OpLog，减少网络往返 +- 监控乱序频率，超过阈值时触发全量同步 + +**详细设计请参考**：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md` + +### 8. key_sequence_map_ 内存清理策略 + +**问题**：Standby 端的 `OpLogApplier` 中，`key_sequence_map_` 用于跟踪每个 key 的 `key_sequence_id`。当 metadata 被删除后，这些条目仍然保留用于乱序检测，长期运行可能导致内存泄漏。 + +**解决方案**：实现定期清理机制： +1. **清理条件**：最后一次操作是 `REMOVE` 且距离当前超过 1 小时 +2. **清理频率**：每小时扫描一次 +3. **保留策略**：`PUT_END` 和 `PUT_REVOKE` 操作的 key 不清理（metadata 可能仍存在） + +**关键设计**： +- 在 `ApplyOpLogEntry` 中触发清理检查，无需额外线程 +- 只清理 `REMOVE` 操作且超过 1 小时的条目 +- 1 小时的时间窗口足够处理网络延迟、重传等异常情况 +- 有效控制内存占用，从潜在的 90MB+ 降低到约 `(活跃key数量 + 1万) × 90字节` + +**详细设计请参考**：`doc/zh/rfc-oplog-key-sequence-map-cleanup.md` + +## 与现有方案对比 + +| 特性 | 当前方案（gRPC 推送） | etcd Watch 方案 | +|------|----------------------|----------------| +| **时序保证** | 依赖网络顺序 | etcd 保证顺序 | +| **可靠性** | 需要 ACK 机制 | etcd 保证可靠性 | +| **断点续传** | 需要实现 | etcd 原生支持 | +| **数据持久化** | 需要额外实现 | etcd 自动持久化 | +| **快照集成** | 需要额外实现 | 易于集成 | +| **实现复杂度** | 高 | 中等 | + +## 实施计划 + +详细的实施计划、优先级和时间估算请参考：`doc/zh/rfc-oplog-implementation-plan.md` + +**实施阶段总览**： +- **Phase 1**：基础框架（P0，2-3 周） +- **Phase 2**：Standby 服务集成（P0，2-3 周） +- **Phase 3**：时序保证和容错（P1，2-3 周） +- **Phase 4**：快照集成和清理（P2，1-2 周） +- **Phase 5**：优化和完善（P3，1-2 周） + +**总计**：8-13 周（约 2-3 个月） + +## 总结 + +本方案利用 etcd 的强一致性和 Watch 机制，实现了可靠的 OpLog 同步。通过只记录 PUT 和 DELETE 事件，大幅减少了 OpLog 大小。通过全局和 key 级别的序列号，保证了时序性。通过与快照机制集成，实现了高效的 OpLog 清理。 + diff --git a/doc/zh/rfc-standby-no-response-handling.md b/doc/zh/rfc-standby-no-response-handling.md new file mode 100644 index 0000000000..a46ab68d8d --- /dev/null +++ b/doc/zh/rfc-standby-no-response-handling.md @@ -0,0 +1,355 @@ +# Standby Master 无响应处理方案 + +## 问题分析 + +当 OpLog 从 Primary Master 同步到 Standby Master 时，如果 Standby 一直不响应，会导致以下问题： + +### 1. **内存压力** +- `OpLogManager` 的 buffer 有上限（`kMaxBufferEntries_ = 100000`），但即使有上限，也可能导致： + - 内存占用持续增长 + - 无法及时 truncate，导致 buffer 长期占用 + - 如果多个 Standby 都无响应，问题会放大 + +### 2. **数据丢失风险** +- 如果 buffer 满了，最老的 OpLog 会被丢弃（`pop_front()`） +- 如果 Standby 后来恢复，可能无法完整同步历史数据 + +### 3. **性能影响** +- 持续尝试发送失败的消息会消耗 CPU +- 阻塞其他正常 Standby 的同步（如果实现不当） + +### 4. **故障检测缺失** +- 当前实现无法区分： + - **网络分区**：Standby 节点正常，但网络不通 + - **节点故障**：Standby 节点宕机 + - **处理慢**：Standby 节点正常，但处理速度慢 + +## 解决方案设计 + +### 方案 1: 超时检测 + 故障隔离（推荐） + +#### 1.1 添加超时检测机制 + +```cpp +struct StandbyState { + std::shared_ptr stream; + uint64_t acked_seq_id{0}; + std::chrono::steady_clock::time_point last_ack_time; + std::chrono::steady_clock::time_point last_send_time; // 新增 + std::vector pending_batch; + + // 新增：超时和重试状态 + enum class State { + HEALTHY, // 正常状态 + SLOW, // 响应慢，但还在处理 + TIMEOUT, // 超时，可能故障 + DISCONNECTED // 已断开连接 + }; + State state{State::HEALTHY}; + uint32_t consecutive_failures{0}; // 连续失败次数 +}; +``` + +#### 1.2 实现超时检测逻辑 + +```cpp +class ReplicationService { +private: + // 配置参数 + static constexpr uint32_t kAckTimeoutMs = 5000; // ACK 超时时间（5秒） + static constexpr uint32_t kSendTimeoutMs = 3000; // 发送超时时间（3秒） + static constexpr uint32_t kMaxConsecutiveFailures = 3; // 最大连续失败次数 + static constexpr uint32_t kHealthCheckIntervalMs = 1000; // 健康检查间隔（1秒） + + // 定期检查 Standby 健康状态 + void CheckStandbyHealth(); + + // 标记 Standby 为故障状态 + void MarkStandbyUnhealthy(const std::string& standby_id); + + // 尝试恢复 Standby 连接 + void TryRecoverStandby(const std::string& standby_id); +}; +``` + +#### 1.3 故障隔离策略 + +**策略 A: 暂停发送（推荐）** +- 当 Standby 超时或连续失败时，暂停向该 Standby 发送新的 OpLog +- 继续向其他健康的 Standby 发送 +- 保留该 Standby 的 `acked_seq_id`，等待恢复后从断点继续 + +**策略 B: 降级处理** +- 将 Standby 标记为 `SLOW` 状态 +- 降低发送频率（例如：每 10 个 OpLog 发送一次） +- 如果持续超时，再升级为 `TIMEOUT` 状态 + +#### 1.4 实现示例 + +```cpp +void ReplicationService::CheckStandbyHealth() { + std::unique_lock lock(mutex_); + auto now = std::chrono::steady_clock::now(); + + for (auto& [standby_id, state] : standbys_) { + // 检查连接状态 + if (!state.stream || !state.stream->IsConnected()) { + state.state = StandbyState::State::DISCONNECTED; + continue; + } + + // 检查 ACK 超时 + auto ack_age = std::chrono::duration_cast( + now - state.last_ack_time).count(); + + if (ack_age > kAckTimeoutMs) { + state.consecutive_failures++; + + if (state.consecutive_failures >= kMaxConsecutiveFailures) { + state.state = StandbyState::State::TIMEOUT; + LOG(WARNING) << "Standby " << standby_id + << " marked as TIMEOUT after " + << state.consecutive_failures << " failures"; + // 暂停向该 Standby 发送 + } else { + state.state = StandbyState::State::SLOW; + LOG(WARNING) << "Standby " << standby_id + << " is slow (ack_age=" << ack_age << "ms)"; + } + } else { + // 恢复正常 + if (state.state != StandbyState::State::HEALTHY) { + LOG(INFO) << "Standby " << standby_id << " recovered"; + state.state = StandbyState::State::HEALTHY; + state.consecutive_failures = 0; + } + } + } +} + +void ReplicationService::BroadcastEntry(const OpLogEntry& entry) { + std::shared_lock lock(mutex_); + + for (auto& [standby_id, state] : standbys_) { + // 跳过故障的 Standby + if (state.state == StandbyState::State::TIMEOUT || + state.state == StandbyState::State::DISCONNECTED) { + continue; + } + + state.pending_batch.push_back(entry); + + if (state.pending_batch.size() >= kBatchSize) { + SendBatch(standby_id, state.pending_batch); + state.pending_batch.clear(); + } + } +} +``` + +### 方案 2: 真正的 ACK 机制 + +当前实现中，`acked_seq_id` 的更新是假设 `Send()` 成功就更新，这是不正确的。应该： + +1. **发送时记录待确认的序列号** +2. **等待 Standby 的 ACK 响应** +3. **只有收到 ACK 后才更新 `acked_seq_id`** + +```cpp +struct StandbyState { + // ... + std::map pending_acks; // seq_id -> send_time + uint64_t last_sent_seq_id{0}; // 最后发送的序列号 +}; + +void ReplicationService::SendBatch(const std::string& standby_id, + const std::vector& entries) { + // ... 发送逻辑 ... + + if (success && !entries.empty()) { + uint64_t last_seq = entries.back().sequence_id; + state.last_sent_seq_id = last_seq; + // 记录待确认的序列号 + state.pending_acks[last_seq] = std::chrono::steady_clock::now(); + // 注意：这里不更新 acked_seq_id，等收到 ACK 再更新 + } +} + +// 处理 Standby 的 ACK 响应 +void ReplicationService::OnAck(const std::string& standby_id, uint64_t acked_seq_id) { + std::unique_lock lock(mutex_); + auto it = standbys_.find(standby_id); + if (it == standbys_.end()) { + return; + } + + auto& state = it->second; + if (acked_seq_id > state.acked_seq_id) { + state.acked_seq_id = acked_seq_id; + state.last_ack_time = std::chrono::steady_clock::now(); + state.consecutive_failures = 0; // 重置失败计数 + + // 清理已确认的 pending_acks + auto ack_it = state.pending_acks.begin(); + while (ack_it != state.pending_acks.end()) { + if (ack_it->first <= acked_seq_id) { + ack_it = state.pending_acks.erase(ack_it); + } else { + ++ack_it; + } + } + } +} +``` + +### 方案 3: 流控（Backpressure）机制 + +如果 Standby 处理慢，应该限制发送速度，避免 Standby 内存溢出： + +```cpp +struct StandbyState { + // ... + size_t in_flight_bytes{0}; // 正在传输的字节数 + size_t max_in_flight_bytes{10 * 1024 * 1024}; // 最大 10MB + uint32_t pending_batch_count{0}; // 待确认的批次数量 + uint32_t max_pending_batches{10}; // 最大待确认批次 +}; + +bool ReplicationService::CanSendToStandby(const StandbyState& state) const { + // 检查流控条件 + if (state.in_flight_bytes >= state.max_in_flight_bytes) { + return false; // 超过流量限制 + } + if (state.pending_batch_count >= state.max_pending_batches) { + return false; // 超过批次限制 + } + return true; +} +``` + +### 方案 4: OpLog Truncate 策略 + +只有当**所有健康的 Standby** 都 ACK 了某个序列号后，才能安全地 truncate： + +```cpp +uint64_t ReplicationService::GetMinAckedSequenceId() const { + std::shared_lock lock(mutex_); + + if (standbys_.empty()) { + // 没有 Standby，可以 truncate 所有 + return oplog_manager_.GetLastSequenceId(); + } + + uint64_t min_acked = UINT64_MAX; + for (const auto& [standby_id, state] : standbys_) { + // 只考虑健康的 Standby + if (state.state == StandbyState::State::HEALTHY || + state.state == StandbyState::State::SLOW) { + min_acked = std::min(min_acked, state.acked_seq_id); + } + } + + return (min_acked == UINT64_MAX) ? 0 : min_acked; +} + +// 定期调用，清理已确认的 OpLog +void ReplicationService::TruncateOpLog() { + uint64_t min_acked = GetMinAckedSequenceId(); + if (min_acked > 0) { + oplog_manager_.TruncateBefore(min_acked); + } +} +``` + +### 方案 5: 重连和恢复机制 + +当 Standby 恢复后，应该能够从断点继续同步： + +```cpp +void ReplicationService::TryRecoverStandby(const std::string& standby_id) { + std::unique_lock lock(mutex_); + auto it = standbys_.find(standby_id); + if (it == standbys_.end()) { + return; + } + + auto& state = it->second; + + // 检查连接是否恢复 + if (state.stream && state.stream->IsConnected()) { + // 从上次 ACK 的位置开始重新发送 + uint64_t start_seq = state.acked_seq_id + 1; + auto entries = oplog_manager_.GetEntriesSince(start_seq, 1000); + + if (!entries.empty()) { + LOG(INFO) << "Recovering Standby " << standby_id + << " from seq_id=" << start_seq + << ", entries=" << entries.size(); + SendBatch(standby_id, entries); + state.state = StandbyState::State::HEALTHY; + } + } +} +``` + +## 实施优先级 + +### Phase 1: 基础超时检测（必须） +1. 添加 `StandbyState::State` 枚举 +2. 实现 `CheckStandbyHealth()` 定期检查 +3. 在 `BroadcastEntry()` 中跳过故障 Standby +4. 添加配置参数（超时时间、最大失败次数） + +### Phase 2: 真正的 ACK 机制（重要） +1. 修改 `SendBatch()` 不立即更新 `acked_seq_id` +2. 添加 `OnAck()` 方法处理 ACK 响应 +3. 实现 `pending_acks` 跟踪机制 + +### Phase 3: 流控和 Truncate（优化） +1. 实现流控机制 +2. 实现安全的 OpLog truncate +3. 添加监控指标（replication lag、failure rate） + +### Phase 4: 恢复机制（完善） +1. 实现重连检测 +2. 实现断点续传 +3. 添加恢复日志 + +## 配置参数建议 + +```cpp +struct ReplicationConfig { + uint32_t ack_timeout_ms = 5000; // ACK 超时时间 + uint32_t send_timeout_ms = 3000; // 发送超时时间 + uint32_t max_consecutive_failures = 3; // 最大连续失败次数 + uint32_t health_check_interval_ms = 1000; // 健康检查间隔 + size_t max_in_flight_bytes = 10 * 1024 * 1024; // 最大传输字节数 + uint32_t max_pending_batches = 10; // 最大待确认批次 + bool enable_backpressure = true; // 是否启用流控 +}; +``` + +## 监控指标 + +建议添加以下监控指标： + +1. **Replication Lag**: 每个 Standby 的延迟（`primary_seq_id - acked_seq_id`） +2. **Failure Rate**: Standby 的失败率 +3. **Timeout Count**: 超时次数 +4. **Recovery Count**: 恢复次数 +5. **OpLog Buffer Size**: OpLog buffer 当前大小 +6. **Truncate Rate**: OpLog truncate 频率 + +## 总结 + +Standby 无响应是一个复杂的分布式系统问题，需要多层次的解决方案： + +1. **超时检测**：及时发现故障 +2. **故障隔离**：避免影响其他 Standby +3. **真正的 ACK**：准确跟踪同步进度 +4. **流控**：保护 Standby 不被压垮 +5. **安全 Truncate**：避免数据丢失 +6. **恢复机制**：支持断点续传 + +建议先实施 Phase 1 和 Phase 2，这两个是最关键的。 + diff --git a/doc/zh/rfc-standby-promotion-lease-initialization.md b/doc/zh/rfc-standby-promotion-lease-initialization.md new file mode 100644 index 0000000000..b10daa8921 --- /dev/null +++ b/doc/zh/rfc-standby-promotion-lease-initialization.md @@ -0,0 +1,439 @@ +# Standby 提升为 Primary 时的 Lease 初始化方案 + +## 问题描述 + +当 Standby Master 被提升为 Primary Master 时，存在一个关键问题：**所有对象的 lease 都是 0（已过期）**。 + +### 问题根源 + +1. **OpLog 中只包含 PUT 和 DELETE 事件** + - `PUT_END` 事件：在 Primary 上创建对象时，`lease_timeout` 被初始化为 0（立即过期） + - `DELETE` 事件：删除对象 + - **不包含** `LEASE_RENEW` 事件（已从 OpLog 中移除） + +2. **Standby 上的对象状态** + - Standby 从 Primary 同步 OpLog，只收到 `PUT_END` 事件 + - 因此 Standby 上所有对象的 `lease_timeout` 都是 0（epoch） + - Standby 不执行驱逐，所以不会检查 lease 是否过期 + +3. **提升为 Primary 后的影响** + - 新 Primary 开始执行驱逐逻辑 + - 由于所有对象的 `lease_timeout` 都是 0，所有对象都会立即被判定为过期 + - 这会导致所有对象被立即驱逐，系统无法正常工作 + +### 问题场景示例 + +``` +时间线： +1. Primary: PutEnd(key="obj1") → lease_timeout = 0 +2. Primary: ExistKey(key="obj1") → lease_timeout = now + 5s (续约) +3. Standby: 同步 PUT_END 事件 → lease_timeout = 0 (没有续约信息) +4. Primary 崩溃 +5. Standby 提升为 Primary +6. 新 Primary: 执行驱逐 → 所有对象 lease_timeout = 0 → 全部被驱逐 ❌ +``` + +## 解决方案 + +### 方案：在提升时给所有对象授予默认租约 + +**核心思路**：当 Standby 被提升为 Primary 时，遍历所有 metadata，给每个对象授予一个默认的租约时间。 + +### 实现设计 + +#### 1. 在 `HotStandbyService::Promote()` 中添加 Lease 初始化逻辑 + +```cpp +std::unique_ptr HotStandbyService::Promote() { + if (!IsReadyForPromotion()) { + LOG(ERROR) << "Standby is not ready for promotion"; + return nullptr; + } + + LOG(INFO) << "Promoting Standby to Primary. Applied seq_id: " + << applied_seq_id_.load(); + + // Stop replication + Stop(); + + // 1. 创建新的 MasterService 实例 + auto master_service = std::make_unique(/* config */); + + // 2. 从 metadata_store_ 恢复 metadata 到新的 MasterService + RestoreMetadataToMasterService(*master_service); + + // 3. 【关键】给所有对象授予默认租约 + InitializeLeasesForAllObjects(*master_service); + + // 4. 执行一次完整的驱逐清理（清理真正过期的对象） + PerformFullEvictionCleanup(*master_service); + + LOG(INFO) << "Standby promoted to Primary successfully"; + return master_service; +} +``` + +#### 2. 实现 `InitializeLeasesForAllObjects()` + +```cpp +void HotStandbyService::InitializeLeasesForAllObjects(MasterService& master_service) { + LOG(INFO) << "Initializing leases for all objects after promotion"; + + uint64_t default_lease_ttl = master_service.GetDefaultLeaseTtl(); + uint64_t default_soft_pin_ttl = master_service.GetDefaultSoftPinTtl(); + + size_t initialized_count = 0; + + // 遍历所有 shard 中的所有 metadata + for (auto& shard : master_service.GetMetadataShards()) { + std::unique_lock lock(shard.mutex); + + for (auto& [key, metadata] : shard.metadata) { + // 检查 lease 是否过期（lease_timeout = 0 表示过期） + if (metadata.IsLeaseExpired()) { + // 授予默认租约 + metadata.GrantLease(default_lease_ttl, default_soft_pin_ttl); + initialized_count++; + + VLOG(2) << "Initialized lease for key: " << key + << ", lease_ttl=" << default_lease_ttl; + } + } + } + + LOG(INFO) << "Initialized leases for " << initialized_count + << " objects after promotion"; +} +``` + +#### 3. 实现 `PerformFullEvictionCleanup()` + +```cpp +void HotStandbyService::PerformFullEvictionCleanup(MasterService& master_service) { + LOG(INFO) << "Performing full eviction cleanup after promotion"; + + // 执行一次完整的驱逐，清理真正过期的对象 + // 注意：此时所有对象的 lease 都已经初始化，只有真正过期的对象才会被驱逐 + master_service.BatchEvict(); + + LOG(INFO) << "Full eviction cleanup completed"; +} +``` + +### 关键设计点 + +#### 1. 默认租约时间的选择 + +**选项 A：使用配置的 `default_kv_lease_ttl`** +- **优点**：简单，与正常操作一致 +- **缺点**：可能给已经很久没有访问的对象也授予租约，导致内存浪费 + +**选项 B：使用较短的租约时间（如 1-2 秒）** +- **优点**：快速淘汰真正不活跃的对象 +- **缺点**：可能误杀活跃对象 + +**推荐：选项 A（使用 `default_kv_lease_ttl`）** + +**理由**： +1. 保守策略，避免误杀活跃对象 +2. 如果对象真的不活跃，会在下次驱逐时被清理 +3. 与正常操作一致，行为可预测 + +#### 2. 何时执行 Lease 初始化 + +**时机**：在 `Promote()` 方法中，在恢复 metadata 之后、开始服务请求之前 + +**流程**： +``` +1. 停止 Standby 的复制循环 +2. 创建新的 MasterService 实例 +3. 恢复 metadata 到新的 MasterService +4. 【关键】初始化所有对象的 lease +5. 执行一次完整的驱逐清理 +6. 开始服务请求 +``` + +#### 3. 与驱逐清理的配合 + +**问题**：如果先初始化 lease，再执行驱逐，那么所有对象都有 lease，不会被驱逐？ + +**解答**： +- 初始化 lease 的目的是**防止误杀活跃对象** +- 驱逐清理的目的是**清理真正过期的对象**（基于 `put_start_time` 等条件） +- 实际上，在 Standby 提升时，所有对象都是"新"的（从 OpLog 恢复），所以应该都保留 +- 如果某些对象在 Primary 崩溃前就已经过期，那么它们应该已经被 Primary 驱逐并产生 DELETE 事件，Standby 上不应该有这些对象 + +**更准确的驱逐逻辑**： +- 在 Standby 提升时，不应该基于 lease 进行驱逐 +- 应该基于其他条件（如 `put_start_time` + `put_start_release_timeout_sec_`）进行清理 +- 或者，在提升时**不执行驱逐**，让正常的驱逐循环来处理 + +**修正后的方案**： + +```cpp +void HotStandbyService::Promote() { + // ... 前面的步骤 ... + + // 3. 给所有对象授予默认租约 + InitializeLeasesForAllObjects(*master_service); + + // 4. 【可选】执行一次清理，但只清理明显无效的对象 + // 注意：不基于 lease 进行驱逐，因为所有对象的 lease 都是 0 + // 可以清理：put_start_time 过期的对象、没有完整 replica 的对象等 + CleanupInvalidObjects(*master_service); + + // 5. 启动 MasterService 的驱逐循环 + // 正常的驱逐循环会基于 lease 和其他条件进行驱逐 +} +``` + +## 实现细节 + +### 1. 在 `MasterService` 中添加辅助方法 + +```cpp +class MasterService { +public: + // 获取默认租约 TTL + uint64_t GetDefaultLeaseTtl() const { return default_kv_lease_ttl_; } + + // 获取默认 Soft Pin TTL + uint64_t GetDefaultSoftPinTtl() const { return default_kv_soft_pin_ttl_; } + + // 获取 metadata shards（用于遍历） + std::vector& GetMetadataShards() { return metadata_shards_; } + + // ... 其他方法 ... +}; +``` + +### 2. 在 `HotStandbyService` 中实现 Lease 初始化 + +```cpp +class HotStandbyService { +private: + void InitializeLeasesForAllObjects(MasterService& master_service); + void CleanupInvalidObjects(MasterService& master_service); + + // ... 其他成员 ... +}; + +void HotStandbyService::InitializeLeasesForAllObjects(MasterService& master_service) { + LOG(INFO) << "Initializing leases for all objects after promotion"; + + uint64_t default_lease_ttl = master_service.GetDefaultLeaseTtl(); + uint64_t default_soft_pin_ttl = master_service.GetDefaultSoftPinTtl(); + + size_t initialized_count = 0; + size_t skipped_count = 0; + + // 遍历所有 shard + for (auto& shard : master_service.GetMetadataShards()) { + std::unique_lock lock(shard.mutex); + + for (auto& [key, metadata] : shard.metadata) { + // 只初始化 lease 为 0 的对象 + if (metadata.IsLeaseExpired()) { + metadata.GrantLease(default_lease_ttl, default_soft_pin_ttl); + initialized_count++; + } else { + // 如果 lease 已经有效，说明可能是从快照恢复的，保留原值 + skipped_count++; + } + } + } + + LOG(INFO) << "Lease initialization completed: " + << initialized_count << " objects initialized, " + << skipped_count << " objects skipped"; +} +``` + +### 3. 清理无效对象（可选） + +```cpp +void HotStandbyService::CleanupInvalidObjects(MasterService& master_service) { + LOG(INFO) << "Cleaning up invalid objects after promotion"; + + size_t cleaned_count = 0; + auto now = std::chrono::steady_clock::now(); + + // 遍历所有 shard + for (auto& shard : master_service.GetMetadataShards()) { + std::unique_lock lock(shard.mutex); + + auto it = shard.metadata.begin(); + while (it != shard.metadata.end()) { + auto& [key, metadata] = *it; + + // 清理条件： + // 1. put_start_time 过期且没有完整 replica + // 2. 所有 replica 都无效 + bool should_cleanup = false; + + if (!metadata.HasCompletedReplicas() && + metadata.put_start_time + + master_service.GetPutStartReleaseTimeout() < now) { + should_cleanup = true; + } else if (!metadata.IsValid()) { + should_cleanup = true; + } + + if (should_cleanup) { + VLOG(1) << "Cleaning up invalid object: " << key; + it = shard.metadata.erase(it); + cleaned_count++; + } else { + ++it; + } + } + } + + LOG(INFO) << "Cleaned up " << cleaned_count << " invalid objects"; +} +``` + +## 边界情况处理 + +### 1. 从快照恢复的场景 + +**场景**：Standby 从快照恢复，快照中可能包含 lease 信息 + +**处理**： +- 如果快照中包含 lease 信息，保留原值 +- 如果快照中 lease 为 0，则初始化 + +**实现**： +```cpp +if (metadata.IsLeaseExpired()) { + // lease 为 0，需要初始化 + metadata.GrantLease(default_lease_ttl, default_soft_pin_ttl); +} else { + // lease 已有效，可能是从快照恢复的，保留原值 + skipped_count++; +} +``` + +### 2. 提升过程中的并发访问 + +**场景**：提升过程中，可能有其他线程访问 metadata + +**处理**： +- 使用 `std::unique_lock` 保护每个 shard +- 提升过程应该是原子的（停止 Standby，创建 Primary） + +### 3. 提升失败的处理 + +**场景**：提升过程中发生错误 + +**处理**： +- 记录错误日志 +- 返回 `nullptr`，表示提升失败 +- Standby 继续运行，等待下次提升机会 + +## 性能考虑 + +### 1. 遍历所有对象的开销 + +**影响**： +- 如果对象数量很大（如 100 万），遍历所有对象可能需要几秒 + +**优化**： +- 使用多线程并行处理不同 shard +- 批量处理，减少锁竞争 + +**实现**： +```cpp +void HotStandbyService::InitializeLeasesForAllObjects(MasterService& master_service) { + auto& shards = master_service.GetMetadataShards(); + + // 并行处理所有 shard + std::vector threads; + for (size_t i = 0; i < shards.size(); ++i) { + threads.emplace_back([&shards, i, &master_service]() { + auto& shard = shards[i]; + std::unique_lock lock(shard.mutex); + + for (auto& [key, metadata] : shard.metadata) { + if (metadata.IsLeaseExpired()) { + metadata.GrantLease( + master_service.GetDefaultLeaseTtl(), + master_service.GetDefaultSoftPinTtl()); + } + } + }); + } + + for (auto& t : threads) { + t.join(); + } +} +``` + +### 2. 提升时间窗口 + +**影响**： +- 提升过程需要时间，期间系统不可用 + +**优化**： +- 尽量减少提升时间 +- 可以考虑在 Standby 阶段就预先初始化 lease（但这样 Standby 也需要维护 lease） + +## 测试场景 + +### 1. 正常提升场景 + +``` +1. Standby 同步了 1000 个对象的 PUT_END 事件 +2. 所有对象的 lease_timeout = 0 +3. Primary 崩溃 +4. Standby 提升为 Primary +5. 验证：所有对象的 lease_timeout > now +6. 验证：系统可以正常服务请求 +``` + +### 2. 从快照恢复的场景 + +``` +1. Standby 从快照恢复，快照中包含 lease 信息 +2. 部分对象的 lease_timeout > 0（从快照恢复） +3. 部分对象的 lease_timeout = 0（新同步的） +4. Standby 提升为 Primary +5. 验证：lease_timeout = 0 的对象被初始化 +6. 验证：lease_timeout > 0 的对象保留原值 +``` + +### 3. 大量对象的场景 + +``` +1. Standby 同步了 100 万个对象 +2. Standby 提升为 Primary +3. 验证：所有对象的 lease 都被初始化 +4. 验证：提升时间在可接受范围内（< 10 秒） +``` + +## 总结 + +### 核心方案 + +**在 Standby 提升为 Primary 时，给所有 lease 为 0 的对象授予默认租约时间** + +### 关键点 + +1. **时机**：在 `Promote()` 中，恢复 metadata 之后、开始服务之前 +2. **租约时间**：使用 `default_kv_lease_ttl`（保守策略） +3. **清理**：可选，清理明显无效的对象（不基于 lease） +4. **性能**：并行处理多个 shard，减少提升时间 + +### 优势 + +1. **简单可靠**：逻辑清晰，易于实现和测试 +2. **保守策略**：避免误杀活跃对象 +3. **与现有机制兼容**：使用现有的 `GrantLease` 方法 + +### 注意事项 + +1. **提升时间**：如果对象数量很大，提升可能需要几秒 +2. **内存影响**：给所有对象授予租约，可能暂时保留一些不活跃对象 +3. **后续清理**：正常的驱逐循环会在后续清理不活跃对象 + diff --git a/doc/zh/rfc-standby-service-integration.md b/doc/zh/rfc-standby-service-integration.md new file mode 100644 index 0000000000..fa16d2cafa --- /dev/null +++ b/doc/zh/rfc-standby-service-integration.md @@ -0,0 +1,673 @@ +# Standby 服务集成方案 + +## 问题描述 + +在现有代码实现中，Standby Master 在 `MasterServiceSupervisor::Start()` 中只是阻塞等待 leader 失效（`WatchUntilDeleted`），没有运行 Standby 服务来同步 OpLog 和恢复 metadata。 + +### 现有代码的问题 + +```cpp +// MasterServiceSupervisor::Start() +mv_helper.ElectLeader(config_.local_hostname, view_version, lease_id); +// 这里会阻塞等待 leader 失效，期间 Standby 什么都不做 +``` + +**问题**： +1. Standby 在等待期间不执行任何操作 +2. 没有 watch etcd 的 OpLog +3. 没有实时恢复 metadata +4. 提升为 Primary 时，metadata 可能不完整 + +### 我们方案的需求 + +根据基于 etcd 的 OpLog 同步方案，Standby 需要： +1. **Watch etcd 的 OpLog**：实时接收 Primary 写入的 OpLog 事件 +2. **实时恢复 metadata**：将 OpLog 应用到本地 metadata store +3. **在等待选举期间持续运行**：即使不是 leader，也要保持数据同步 + +## 解决方案 + +### 核心思路 + +**在 Standby 模式下并行运行 Standby 服务**： +- 检测到有 leader 时，启动 Standby 服务 +- Standby 服务 watch etcd OpLog 并实时恢复 metadata +- 选举成功后，停止 Standby 服务并提升为 Primary + +### 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ MasterServiceSupervisor::Start() │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 1. 检查当前是否有 leader │ +│ - GetMasterView() │ +│ - 如果有 leader 且不是自己 → Standby 模式 │ +│ - 如果没有 leader → 直接选举 │ +└─────────────────────────────────────────────────────────┘ + │ + ├─ 有 leader (Standby 模式) + │ │ + │ ▼ + │ ┌─────────────────────────────────────────────┐ + │ │ 2. 启动 Standby 服务 │ + │ │ - 创建 HotStandbyService │ + │ │ - 启动 ReplicationLoop (watch etcd) │ + │ │ - 启动 VerificationLoop │ + │ └─────────────────────────────────────────────┘ + │ │ + │ ▼ + │ ┌─────────────────────────────────────────────┐ + │ │ 3. 阻塞等待 leader 失效 │ + │ │ - ElectLeader() (WatchUntilDeleted) │ + │ │ - 期间 Standby 服务持续运行 │ + │ └─────────────────────────────────────────────┘ + │ + └─ 没有 leader (直接选举) + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 4. 选举成功 │ +│ - 停止 Standby 服务（如果正在运行） │ +│ - 检查是否准备好提升 │ +│ - 等待 5 秒防止 split-brain │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 5. 提升为 Primary │ +│ - 调用 HotStandbyService::Promote() │ +│ - 初始化所有对象的 lease │ +│ - 创建 WrappedMasterService │ +│ - 启动 RPC 服务器 │ +└─────────────────────────────────────────────────────────┘ +``` + +## 实现设计 + +### 1. 修改 MasterServiceSupervisor::Start() + +```cpp +int MasterServiceSupervisor::Start() { + while (true) { + LOG(INFO) << "Init master service..."; + coro_rpc::coro_rpc_server server( + config_.rpc_thread_num, config_.rpc_port, config_.rpc_address, + config_.rpc_conn_timeout, config_.rpc_enable_tcp_no_delay); + const char* value = std::getenv("MC_RPC_PROTOCOL"); + if (value && std::string_view(value) == "rdma") { + server.init_ibv(); + } + + LOG(INFO) << "Init leader election helper..."; + MasterViewHelper mv_helper; + if (mv_helper.ConnectToEtcd(config_.etcd_endpoints) != ErrorCode::OK) { + LOG(ERROR) << "Failed to connect to etcd endpoints: " + << config_.etcd_endpoints; + return -1; + } + + // 【新增】检查当前是否有 leader + ViewVersionId current_version = 0; + std::string current_master; + auto ret = mv_helper.GetMasterView(current_master, current_version); + + // 【新增】如果有 leader 且不是自己，启动 Standby 服务 + std::unique_ptr standby_service = nullptr; + if (ret == ErrorCode::OK && current_master != config_.local_hostname) { + LOG(INFO) << "Current leader: " << current_master + << ", starting Standby service..."; + + // 创建并启动 Standby 服务 + HotStandbyConfig standby_config; + standby_config.standby_id = config_.local_hostname; + standby_config.primary_address = current_master; + standby_config.etcd_endpoints = config_.etcd_endpoints; + standby_config.cluster_id = config_.cluster_id; + standby_config.enable_verification = true; + standby_config.max_replication_lag_entries = 1000; + + standby_service = std::make_unique(standby_config); + auto err = standby_service->Start(current_master); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start Standby service: " << err; + standby_service.reset(); + } else { + LOG(INFO) << "Standby service started, watching OpLog from etcd"; + } + } + + // 尝试选举（如果有 leader，会阻塞等待；如果没有，立即选举） + LOG(INFO) << "Trying to elect self as leader..."; + EtcdLeaseId lease_id = 0; + ViewVersionId view_version = 0; + mv_helper.ElectLeader(config_.local_hostname, view_version, lease_id); + + // 【新增】停止 Standby 服务（如果正在运行） + if (standby_service) { + LOG(INFO) << "Stopping Standby service before promotion..."; + standby_service->Stop(); + + // 【新增】检查是否准备好提升 + if (!standby_service->IsReadyForPromotion()) { + LOG(WARNING) << "Standby is not ready for promotion, " + << "lag: " << standby_service->GetSyncStatus().lag_entries + << " entries, but proceeding anyway due to leader election"; + } + } + + // 防止 split-brain + const int waiting_time = ETCD_MASTER_VIEW_LEASE_TTL; + std::this_thread::sleep_for(std::chrono::seconds(waiting_time)); + + LOG(INFO) << "Starting master service as Primary..."; + + // 【新增】如果 Standby 服务存在，使用它来初始化 MasterService + std::unique_ptr promoted_service = nullptr; + if (standby_service) { + promoted_service = standby_service->Promote(); + if (!promoted_service) { + LOG(ERROR) << "Failed to promote Standby to Primary"; + // 继续使用新的 MasterService，但 metadata 可能不完整 + } else { + LOG(INFO) << "Successfully promoted Standby to Primary"; + } + } + + // 创建 WrappedMasterService + // 注意：这里需要将 promoted_service 的 metadata 复制到新的 MasterService + // 或者修改 WrappedMasterService 的构造方式，支持从 promoted_service 初始化 + mooncake::WrappedMasterService wrapped_master_service( + mooncake::WrappedMasterServiceConfig(config_, view_version)); + + // TODO: 如果 promoted_service 存在，需要将其 metadata 复制到 wrapped_master_service + // 这需要修改 WrappedMasterService 或 MasterService 的接口 + + mooncake::RegisterRpcService(server, wrapped_master_service); + + // Start a thread to keep the leader alive + auto keep_leader_thread = + std::thread([&server, &mv_helper, lease_id]() { + mv_helper.KeepLeader(lease_id); + LOG(INFO) << "Trying to stop server..."; + server.stop(); + }); + + async_simple::Future ec = + server.async_start(); + if (ec.hasResult()) { + LOG(ERROR) << "Failed to start master service: " + << ec.result().value(); + auto etcd_err = EtcdHelper::CancelKeepAlive(lease_id); + if (etcd_err != ErrorCode::OK) { + LOG(ERROR) << "Failed to cancel keep leader alive: " + << etcd_err; + } + keep_leader_thread.join(); + return -1; + } + + // Block until the server is stopped + auto server_err = std::move(ec).get(); + LOG(ERROR) << "Master service stopped: " << server_err; + + // If the server is closed due to internal errors, we need to manually + // stop keep leader alive. + auto etcd_err = EtcdHelper::CancelKeepAlive(lease_id); + LOG(INFO) << "Cancel keep leader alive: " << etcd_err; + keep_leader_thread.join(); + } + return 0; +} +``` + +### 2. 修改 HotStandbyService::ReplicationLoop() + +```cpp +void HotStandbyService::ReplicationLoop() { + LOG(INFO) << "Replication loop started"; + + // 【新增】创建 OpLogWatcher（使用 etcd Watch） + OpLogWatcher oplog_watcher( + config_.etcd_endpoints, + config_.cluster_id, + this); // HotStandbyService 作为 OpLogApplier + + // 【新增】从上次处理的 sequence_id 开始读取历史 OpLog + uint64_t start_seq_id = applied_seq_id_.load() + 1; + if (start_seq_id > 1) { + std::vector historical_entries; + if (oplog_watcher.ReadOpLogSince(start_seq_id, historical_entries)) { + LOG(INFO) << "Read " << historical_entries.size() + << " historical OpLog entries from sequence_id " + << start_seq_id; + + // 应用历史 OpLog + for (const auto& entry : historical_entries) { + ApplyOpLogEntry(entry); + } + } else { + LOG(WARNING) << "Failed to read historical OpLog, " + << "may need to perform full snapshot sync"; + } + } + + // 【新增】启动 etcd Watch + oplog_watcher.Start(); + is_connected_.store(true); + LOG(INFO) << "OpLog watcher started, watching etcd for new OpLog entries"; + + while (running_.load()) { + // OpLogWatcher 会在后台线程中处理 Watch 事件 + // 当收到新 OpLog 时，会调用 ApplyOpLogEntry() + + // 定期检查同步状态 + auto status = GetSyncStatus(); + if (status.lag_entries > config_.max_replication_lag_entries) { + LOG(WARNING) << "Replication lag is high: " + << status.lag_entries << " entries"; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + + // 【新增】停止 Watch + oplog_watcher.Stop(); + is_connected_.store(false); + LOG(INFO) << "Replication loop stopped"; +} +``` + +### 3. 实现 OpLogWatcher（基于 etcd Watch） + +```cpp +class OpLogWatcher { +public: + OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, + OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), + cluster_id_(cluster_id), + applier_(applier), + etcd_oplog_store_(etcd_endpoints, cluster_id) { + etcd_prefix_ = "mooncake-store/oplog/" + cluster_id + "/"; + } + + void Start() { + if (running_.load()) { + LOG(WARNING) << "OpLogWatcher is already running"; + return; + } + + running_.store(true); + watch_thread_ = std::thread(&OpLogWatcher::WatchOpLogThreadFunc, this); + LOG(INFO) << "OpLogWatcher started"; + } + + void Stop() { + if (!running_.load()) { + return; + } + + running_.store(false); + if (watch_thread_.joinable()) { + watch_thread_.join(); + } + LOG(INFO) << "OpLogWatcher stopped"; + } + + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries) { + return etcd_oplog_store_.ReadOpLogSince(start_seq_id, 1000, entries); + } + +private: + void WatchOpLogThreadFunc() { + LOG(INFO) << "OpLog watch thread started"; + + // 从上次处理的 sequence_id 开始 Watch + uint64_t start_seq_id = last_processed_sequence_id_ + 1; + std::string watch_prefix = etcd_prefix_; + + while (running_.load()) { + try { + // 使用 etcd Watch 监听 OpLog 变化 + // 这里需要使用 etcd 的 Watch API + // 假设 EtcdHelper 提供了 WatchWithPrefix 方法 + auto watch_result = EtcdHelper::WatchWithPrefix( + watch_prefix.c_str(), + watch_prefix.size(), + [this](const EtcdWatchEvent& event) { + HandleWatchEvent(event); + }); + + if (!watch_result) { + LOG(ERROR) << "Watch failed, retrying..."; + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + } catch (const std::exception& e) { + LOG(ERROR) << "Exception in watch thread: " << e.what(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + } + + LOG(INFO) << "OpLog watch thread stopped"; + } + + void HandleWatchEvent(const EtcdWatchEvent& event) { + if (event.type == EtcdWatchEventType::PUT) { + // 解析 OpLog Entry + OpLogEntry entry; + if (DeserializeOpLogEntry(event.value, entry)) { + // 应用 OpLog + if (applier_->ApplyOpLogEntry(entry)) { + last_processed_sequence_id_ = entry.sequence_id; + VLOG(2) << "Applied OpLog entry: sequence_id=" + << entry.sequence_id + << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + } else { + LOG(WARNING) << "Failed to apply OpLog entry: sequence_id=" + << entry.sequence_id; + } + } else { + LOG(ERROR) << "Failed to deserialize OpLog entry from key: " + << event.key; + } + } else if (event.type == EtcdWatchEventType::DELETE) { + // OpLog 被清理，记录日志 + VLOG(1) << "OpLog entry deleted: " << event.key; + } + } + + std::string etcd_endpoints_; + std::string cluster_id_; + std::string etcd_prefix_; + OpLogApplier* applier_; + EtcdOpLogStore etcd_oplog_store_; + + std::atomic running_{false}; + std::thread watch_thread_; + std::atomic last_processed_sequence_id_{0}; +}; +``` + +### 4. HotStandbyService 实现 OpLogApplier 接口 + +```cpp +class HotStandbyService : public OpLogApplier { +public: + // 实现 OpLogApplier 接口 + bool ApplyOpLogEntry(const OpLogEntry& entry) override { + std::lock_guard lock(mutex_); + + // 检查时序性 + if (!CheckSequenceOrder(entry)) { + LOG(WARNING) << "Sequence order violation for entry: " + << entry.sequence_id; + return false; + } + + // 应用 OpLog + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + default: + LOG(WARNING) << "Unknown OpType: " + << static_cast(entry.op_type); + return false; + } + + applied_seq_id_.store(entry.sequence_id); + return true; + } + +private: + void ApplyPutEnd(const OpLogEntry& entry) { + // 从 metadata_store_ 创建或更新 metadata + // 这里需要实现完整的 metadata 恢复逻辑 + if (metadata_store_) { + metadata_store_->entry_count++; + } + } + + void ApplyPutRevoke(const OpLogEntry& entry) { + // 处理 PUT_REVOKE + // ... + } + + void ApplyRemove(const OpLogEntry& entry) { + // 从 metadata_store_ 删除 metadata + if (metadata_store_ && metadata_store_->entry_count > 0) { + metadata_store_->entry_count--; + } + } + + bool CheckSequenceOrder(const OpLogEntry& entry) { + // 检查全局序列号 + if (entry.sequence_id <= applied_seq_id_.load()) { + LOG(WARNING) << "Received out-of-order entry: " + << "expected > " << applied_seq_id_.load() + << ", got " << entry.sequence_id; + return false; + } + + // 检查 key 级别的序列号 + // 这里需要维护 key_sequence_map_ + // ... + + return true; + } +}; +``` + +## 关键设计点 + +### 1. Standby 服务生命周期 + +``` +启动阶段： +1. 检测到有 leader → 创建 HotStandbyService +2. 启动 ReplicationLoop → 读取历史 OpLog → 启动 Watch +3. 启动 VerificationLoop（可选） + +运行阶段： +1. OpLogWatcher 持续 watch etcd +2. 收到新 OpLog → 调用 ApplyOpLogEntry +3. 实时更新 metadata_store_ + +提升阶段： +1. 选举成功 → 停止 Standby 服务 +2. 调用 Promote() → 初始化 lease → 创建 MasterService +3. 启动 Primary 服务 +``` + +### 2. 历史 OpLog 读取 + +**策略**： +- 从 `applied_seq_id_ + 1` 开始读取 +- 如果 `applied_seq_id_` 为 0，说明是首次启动，需要从快照开始 +- 批量读取（每次 1000 条），避免一次性读取过多 + +**实现**： +```cpp +uint64_t start_seq_id = applied_seq_id_.load() + 1; +if (start_seq_id == 1) { + // 首次启动，需要从快照恢复 + // 或者从 sequence_id = 1 开始读取所有历史 +} +std::vector entries; +oplog_watcher.ReadOpLogSince(start_seq_id, entries); +``` + +### 3. etcd Watch 实现 + +**关键点**： +- 使用 `WatchWithPrefix` 监听 OpLog 前缀 +- 处理 Watch 断开和重连 +- 处理序列号不连续的情况 + +**Watch 前缀**： +``` +mooncake-store/oplog/{cluster_id}/ +``` + +### 4. 时序保证 + +**全局序列号**： +- 检查 `entry.sequence_id > applied_seq_id_` +- 如果序列号不连续，缓存待处理 + +**Key 级别序列号**： +- 维护 `key_sequence_map_` 记录每个 key 的最后 sequence_id +- 检查 `entry.key_sequence_id > key_sequence_map_[key]` + +## 与现有方案的集成 + +### 1. 与快照机制集成 + +**场景**：Standby 首次启动或需要全量同步 + +**流程**： +1. 检测到 `applied_seq_id_ == 0` 或 lag 过大 +2. 请求 Primary 的快照 +3. 应用快照 +4. 从快照的 `last_oplog_sequence_id` 开始读取增量 OpLog + +### 2. 与提升机制集成 + +**流程**： +1. 选举成功 +2. 停止 Standby 服务 +3. 检查同步状态（`IsReadyForPromotion()`） +4. 调用 `Promote()` → 初始化 lease +5. 创建 MasterService 并启动 + +### 3. 与 OpLog 清理集成 + +**场景**：OpLog 被清理后，Watch 可能收到 DELETE 事件 + +**处理**： +- 记录警告日志 +- 如果发现大量 OpLog 被删除，可能需要重新同步 + +## 错误处理和容错 + +### 1. Watch 断开 + +**处理**： +- 自动重连 +- 从上次处理的 sequence_id 重新 Watch +- 如果重连失败，记录错误并重试 + +### 2. 序列号不连续 + +**处理**： +- 缓存待处理的条目 +- 等待一段时间看是否有缺失的条目到达 +- 如果超时，请求 Primary 或从 etcd 读取缺失的条目 + +### 3. 应用失败 + +**处理**： +- 记录错误日志 +- 不更新 `applied_seq_id_` +- 继续处理后续条目（但可能影响一致性） + +## 性能考虑 + +### 1. Watch 性能 + +- etcd Watch 是高效的，不会产生大量网络开销 +- 批量处理 Watch 事件，减少锁竞争 + +### 2. 历史 OpLog 读取 + +- 批量读取（每次 1000 条） +- 并行应用（如果支持） + +### 3. Metadata 更新 + +- 使用适当的锁粒度 +- 考虑使用无锁数据结构（如果可能） + +## 测试场景 + +### 1. 正常 Standby 运行 + +``` +1. 启动 Standby，检测到有 leader +2. 启动 Standby 服务 +3. Watch etcd OpLog +4. 实时应用 OpLog 到 metadata +5. 验证 metadata 与 Primary 一致 +``` + +### 2. Standby 提升为 Primary + +``` +1. Standby 正在运行 +2. Primary 失效 +3. Standby 选举成功 +4. 停止 Standby 服务 +5. 提升为 Primary +6. 验证 metadata 完整性 +``` + +### 3. Watch 断开重连 + +``` +1. Standby 正在 Watch +2. etcd 连接断开 +3. 自动重连 +4. 从上次处理的 sequence_id 继续 +5. 验证没有丢失 OpLog +``` + +### 4. 历史 OpLog 读取 + +``` +1. Standby 重启 +2. applied_seq_id_ = 1000 +3. 读取 sequence_id >= 1001 的历史 OpLog +4. 应用历史 OpLog +5. 启动 Watch 监听新 OpLog +``` + +## 总结 + +### 核心方案 + +**在 Standby 模式下并行运行 Standby 服务，watch etcd OpLog 并实时恢复 metadata** + +### 关键实现 + +1. **MasterServiceSupervisor**：检测 leader，启动/停止 Standby 服务 +2. **HotStandbyService**：实现 OpLogApplier 接口，管理 Standby 生命周期 +3. **OpLogWatcher**：watch etcd OpLog，处理 Watch 事件 +4. **时序保证**：全局和 key 级别的序列号检查 + +### 优势 + +1. **实时同步**：Standby 实时接收并应用 OpLog +2. **数据完整性**：提升时 metadata 已完整 +3. **自动恢复**：Watch 断开自动重连 +4. **与现有方案兼容**：不影响现有的选举和提升逻辑 + +### 注意事项 + +1. **Watch 性能**：需要确保 etcd Watch 的性能 +2. **序列号不连续**：需要处理缺失的 OpLog +3. **Metadata 恢复**：需要完整实现 metadata 的恢复逻辑 +4. **提升时的数据迁移**：需要将 Standby 的 metadata 迁移到 Primary + From 4d693d2f0f8ce2b58704675d7d68a787de4559ab Mon Sep 17 00:00:00 2001 From: BernardLee Date: Sat, 27 Dec 2025 09:04:53 +0800 Subject: [PATCH 10/59] phase 1 --- mooncake-common/etcd/etcd_wrapper.go | 107 +++++++++++ mooncake-store/include/etcd_helper.h | 49 +++++ mooncake-store/include/etcd_oplog_store.h | 154 ++++++++------- mooncake-store/include/oplog_manager.h | 11 ++ mooncake-store/src/CMakeLists.txt | 1 + mooncake-store/src/etcd_helper.cpp | 93 +++++++++ mooncake-store/src/etcd_oplog_store.cpp | 221 ++++++++++++++++++++++ mooncake-store/src/master_service.cpp | 29 +++ mooncake-store/src/oplog_manager.cpp | 23 +++ 9 files changed, 609 insertions(+), 79 deletions(-) create mode 100644 mooncake-store/src/etcd_oplog_store.cpp diff --git a/mooncake-common/etcd/etcd_wrapper.go b/mooncake-common/etcd/etcd_wrapper.go index e57ae53628..ea7aec51e9 100644 --- a/mooncake-common/etcd/etcd_wrapper.go +++ b/mooncake-common/etcd/etcd_wrapper.go @@ -12,6 +12,7 @@ import ( "strings" "sync" "time" + "unsafe" clientv3 "go.etcd.io/etcd/client/v3" ) @@ -425,4 +426,110 @@ func EtcdStoreCancelKeepAliveWrapper(leaseId int64, errMsg **C.char) int { return 0 } +//export EtcdStorePutWrapper +func EtcdStorePutWrapper(key *C.char, keySize C.int, value *C.char, valueSize C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + v := C.GoStringN(value, valueSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := storeClient.Put(ctx, k, v) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + return 0 +} + +//export EtcdStoreGetWithPrefixWrapper +func EtcdStoreGetWithPrefixWrapper(prefix *C.char, prefixSize C.int, keys **C.char, keySizes **C.int, values **C.char, valueSizes **C.int, count *C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := storeClient.Get(ctx, p, clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + if len(resp.Kvs) == 0 { + *count = 0 + return 0 + } + + // Allocate arrays for keys and values + keyCount := len(resp.Kvs) + *count = C.int(keyCount) + + // Allocate memory for arrays + keysArray := (*[1 << 30]*C.char)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof((*C.char)(nil))))) + keySizesArray := (*[1 << 30]C.int)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof(C.int(0))))) + valuesArray := (*[1 << 30]*C.char)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof((*C.char)(nil))))) + valueSizesArray := (*[1 << 30]C.int)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof(C.int(0))))) + + for i, kv := range resp.Kvs { + keysArray[i] = C.CString(string(kv.Key)) + keySizesArray[i] = C.int(len(kv.Key)) + valuesArray[i] = C.CString(string(kv.Value)) + valueSizesArray[i] = C.int(len(kv.Value)) + } + + *keys = (*C.char)(unsafe.Pointer(keysArray)) + *keySizes = (*C.int)(unsafe.Pointer(keySizesArray)) + *values = (*C.char)(unsafe.Pointer(valuesArray)) + *valueSizes = (*C.int)(unsafe.Pointer(valueSizesArray)) + + return 0 +} + +//export EtcdStoreGetFirstKeyWithPrefixWrapper +func EtcdStoreGetFirstKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, firstKey **C.char, firstKeySize *C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + resp, err := storeClient.Get(ctx, p, clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), clientv3.WithLimit(1)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + if len(resp.Kvs) == 0 { + *errMsg = C.CString("no key found with prefix") + return -2 + } + kv := resp.Kvs[0] + *firstKey = C.CString(string(kv.Key)) + *firstKeySize = C.int(len(kv.Key)) + return 0 +} + +//export EtcdStoreDeleteRangeWrapper +func EtcdStoreDeleteRangeWrapper(startKey *C.char, startKeySize C.int, endKey *C.char, endKeySize C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + start := C.GoStringN(startKey, startKeySize) + end := C.GoStringN(endKey, endKeySize) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := storeClient.Delete(ctx, start, clientv3.WithRange(end)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + // resp.Deleted contains the number of deleted keys + return 0 +} + func main() {} diff --git a/mooncake-store/include/etcd_helper.h b/mooncake-store/include/etcd_helper.h index 1f272142ac..f4a2a51752 100644 --- a/mooncake-store/include/etcd_helper.h +++ b/mooncake-store/include/etcd_helper.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include "types.h" @@ -90,6 +92,53 @@ class EtcdHelper { */ static ErrorCode CancelKeepAlive(EtcdLeaseId lease_id); + /* + * @brief Put a key-value pair to etcd. + * @param key: The key to put. + * @param key_size: The size of the key in bytes. + * @param value: The value to put. + * @param value_size: The size of the value in bytes. + * @return: Error code. + */ + static ErrorCode Put(const char* key, const size_t key_size, + const char* value, const size_t value_size); + + /* + * @brief Get all key-value pairs with a given prefix. + * @param prefix: The prefix to search for. + * @param prefix_size: The size of the prefix in bytes. + * @param keys: Output param, vector of keys. + * @param values: Output param, vector of values. + * @return: Error code. + */ + static ErrorCode GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values); + + /* + * @brief Get the first key with a given prefix (sorted by key). + * @param prefix: The prefix to search for. + * @param prefix_size: The size of the prefix in bytes. + * @param first_key: Output param, the first key found. + * @return: Error code. ETCD_KEY_NOT_EXIST if no key found. + */ + static ErrorCode GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key); + + /* + * @brief Delete a range of keys from etcd. + * @param start_key: The start key (inclusive). + * @param start_key_size: The size of the start key in bytes. + * @param end_key: The end key (exclusive). + * @param end_key_size: The size of the end key in bytes. + * @return: Error code. + */ + static ErrorCode DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size); + private: // Variables that are used to ensure the etcd client // is only connected once. diff --git a/mooncake-store/include/etcd_oplog_store.h b/mooncake-store/include/etcd_oplog_store.h index cbf1f46964..5617736afd 100644 --- a/mooncake-store/include/etcd_oplog_store.h +++ b/mooncake-store/include/etcd_oplog_store.h @@ -5,139 +5,135 @@ #include #include "oplog_manager.h" +#include "types.h" namespace mooncake { /** - * @brief Store OpLog entries to etcd for reliable replication + * @brief Store for OpLog entries in etcd. * - * This class handles writing OpLog entries to etcd and provides methods - * for reading and managing OpLog entries in etcd. + * This class is responsible for writing OpLog entries to etcd and reading them back. + * OpLog entries are stored with keys in the format: + * /oplog/{cluster_id}/{sequence_id} + * + * The latest sequence_id is also stored at: + * /oplog/{cluster_id}/latest */ class EtcdOpLogStore { public: /** - * @brief Constructor - * @param etcd_endpoints Comma-separated etcd endpoints - * @param cluster_id Cluster identifier - */ - EtcdOpLogStore(const std::string& etcd_endpoints, - const std::string& cluster_id); - - ~EtcdOpLogStore(); - - /** - * @brief Write a single OpLog entry to etcd - * @param entry OpLog entry to write - * @return true on success, false on failure + * @brief Constructor. + * @param cluster_id: The cluster ID for this OpLog store. */ - bool WriteOpLog(const OpLogEntry& entry); + explicit EtcdOpLogStore(const std::string& cluster_id); /** - * @brief Write multiple OpLog entries to etcd (batch operation) - * @param entries OpLog entries to write - * @return true on success, false on failure + * @brief Write an OpLog entry to etcd. + * @param entry: The OpLog entry to write. + * @return: Error code. */ - bool WriteOpLogBatch(const std::vector& entries); + ErrorCode WriteOpLog(const OpLogEntry& entry); /** - * @brief Update the latest sequence ID in etcd - * @param sequence_id Latest sequence ID - * @return true on success, false on failure + * @brief Read an OpLog entry from etcd by sequence_id. + * @param sequence_id: The sequence ID of the entry to read. + * @param entry: Output param, the OpLog entry. + * @return: Error code. */ - bool UpdateLatestSequenceId(uint64_t sequence_id); + ErrorCode ReadOpLog(uint64_t sequence_id, OpLogEntry& entry); /** - * @brief Get the latest sequence ID from etcd - * @return Latest sequence ID, or 0 if not found + * @brief Read OpLog entries starting from a given sequence_id. + * @param start_sequence_id: The starting sequence ID (exclusive). + * @param limit: Maximum number of entries to read (default: 1000). + * @param entries: Output param, vector of OpLog entries. + * @return: Error code. */ - uint64_t GetLatestSequenceId() const; + ErrorCode ReadOpLogSince(uint64_t start_sequence_id, size_t limit, + std::vector& entries); /** - * @brief Record snapshot sequence ID - * @param snapshot_id Snapshot identifier - * @param sequence_id Sequence ID at snapshot time - * @return true on success, false on failure + * @brief Get the latest sequence_id from etcd. + * @param sequence_id: Output param, the latest sequence_id. + * @return: Error code. ETCD_KEY_NOT_EXIST if no OpLog exists yet. */ - bool RecordSnapshotSequenceId(const std::string& snapshot_id, - uint64_t sequence_id); + ErrorCode GetLatestSequenceId(uint64_t& sequence_id); /** - * @brief Get snapshot sequence ID - * @param snapshot_id Snapshot identifier - * @return Sequence ID, or 0 if not found + * @brief Update the latest sequence_id in etcd. + * @param sequence_id: The latest sequence_id to update. + * @return: Error code. */ - uint64_t GetSnapshotSequenceId(const std::string& snapshot_id) const; + ErrorCode UpdateLatestSequenceId(uint64_t sequence_id); /** - * @brief Read OpLog entries from etcd since a given sequence ID - * @param start_seq_id Starting sequence ID (exclusive) - * @param limit Maximum number of entries to read - * @param entries Output vector of OpLog entries - * @return true on success, false on failure + * @brief Record the sequence_id corresponding to a snapshot. + * @param snapshot_id: The snapshot ID. + * @param sequence_id: The sequence_id at which the snapshot was taken. + * @return: Error code. */ - bool ReadOpLogSince(uint64_t start_seq_id, size_t limit, - std::vector& entries) const; + ErrorCode RecordSnapshotSequenceId(const std::string& snapshot_id, + uint64_t sequence_id); /** - * @brief Read a single OpLog entry by sequence ID - * @param sequence_id Sequence ID - * @param entry Output OpLog entry - * @return true on success, false on failure + * @brief Get the sequence_id for a given snapshot. + * @param snapshot_id: The snapshot ID. + * @param sequence_id: Output param, the sequence_id. + * @return: Error code. ETCD_KEY_NOT_EXIST if snapshot not found. */ - bool ReadOpLogEntry(uint64_t sequence_id, OpLogEntry& entry) const; + ErrorCode GetSnapshotSequenceId(const std::string& snapshot_id, + uint64_t& sequence_id); /** - * @brief Cleanup OpLog entries before a given sequence ID - * @param sequence_id Sequence ID (entries with seq_id < sequence_id will be deleted) - * @return true on success, false on failure + * @brief Clean up OpLog entries before a given sequence_id. + * @param before_sequence_id: All entries with sequence_id < before_sequence_id + * will be deleted. + * @return: Error code. */ - bool CleanupOpLogBefore(uint64_t sequence_id); + ErrorCode CleanupOpLogBefore(uint64_t before_sequence_id); private: /** - * @brief Build etcd key for OpLog entry - * @param sequence_id Sequence ID - * @return etcd key string + * @brief Build the etcd key for an OpLog entry. + * @param sequence_id: The sequence ID. + * @return: The etcd key. */ std::string BuildOpLogKey(uint64_t sequence_id) const; /** - * @brief Build etcd key for latest sequence ID - * @return etcd key string + * @brief Build the etcd key for the latest sequence_id. + * @return: The etcd key. */ - std::string BuildLatestSequenceIdKey() const; + std::string BuildLatestKey() const; /** - * @brief Build etcd key for snapshot sequence ID - * @param snapshot_id Snapshot identifier - * @return etcd key string + * @brief Build the etcd key for a snapshot sequence_id. + * @param snapshot_id: The snapshot ID. + * @return: The etcd key. */ - std::string BuildSnapshotSequenceIdKey(const std::string& snapshot_id) const; + std::string BuildSnapshotKey(const std::string& snapshot_id) const; /** - * @brief Serialize OpLog entry to JSON string - * @param entry OpLog entry - * @return JSON string + * @brief Serialize an OpLogEntry to JSON string. + * @param entry: The OpLog entry to serialize. + * @return: The JSON string. */ std::string SerializeOpLogEntry(const OpLogEntry& entry) const; /** - * @brief Deserialize OpLog entry from JSON string - * @param data JSON string - * @param entry Output OpLog entry - * @return true on success, false on failure + * @brief Deserialize a JSON string to OpLogEntry. + * @param json_str: The JSON string. + * @param entry: Output param, the OpLog entry. + * @return: true if successful, false otherwise. */ - bool DeserializeOpLogEntry(const std::string& data, - OpLogEntry& entry) const; + bool DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) const; - std::string etcd_endpoints_; std::string cluster_id_; - std::string etcd_prefix_; // e.g., "mooncake-store/oplog" - - // etcd client will be added when implementing - // For now, we use EtcdHelper + static constexpr const char* kOpLogPrefix = "/oplog/"; + static constexpr const char* kLatestSuffix = "/latest"; + static constexpr const char* kSnapshotPrefix = "/oplog/"; + static constexpr const char* kSnapshotSuffix = "/snapshot/"; }; } // namespace mooncake - diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index f971b1e13f..3f680ba130 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -10,6 +11,9 @@ namespace mooncake { +// Forward declaration +class EtcdOpLogStore; + // Operation types for hot-standby replication. // This is a minimal subset that can be extended later. enum class OpType : uint8_t { @@ -42,6 +46,10 @@ class OpLogManager { public: OpLogManager(); + // Set the EtcdOpLogStore for writing OpLog to etcd (optional). + // If not set, OpLog will only be stored in memory buffer. + void SetEtcdOpLogStore(std::shared_ptr etcd_oplog_store); + // Append a new entry and return the assigned sequence_id. uint64_t Append(OpType type, const std::string& key, const std::string& payload = std::string()); @@ -72,6 +80,9 @@ class OpLogManager { // Track per-key sequence ID for ordering guarantee std::unordered_map key_sequence_map_; + // Optional etcd OpLog store for persistent storage + std::shared_ptr etcd_oplog_store_; + // Simple bounds to avoid unbounded memory growth. static constexpr size_t kMaxBufferEntries_ = 100000; }; diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 603e1d52be..8390fc9d59 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -27,6 +27,7 @@ set(MOONCAKE_STORE_SOURCES http_metadata_server.cpp file_storage.cpp oplog_manager.cpp + etcd_oplog_store.cpp // replication_service.cpp removed - using etcd-based OpLog sync instead ) diff --git a/mooncake-store/src/etcd_helper.cpp b/mooncake-store/src/etcd_helper.cpp index 5417de5afc..fcee8043f3 100644 --- a/mooncake-store/src/etcd_helper.cpp +++ b/mooncake-store/src/etcd_helper.cpp @@ -152,6 +152,71 @@ ErrorCode EtcdHelper::CancelKeepAlive(EtcdLeaseId lease_id) { } return ErrorCode::OK; } + +ErrorCode EtcdHelper::Put(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + char* err_msg = nullptr; + int ret = EtcdStorePutWrapper((char*)key, (int)key_size, (char*)value, + (int)value_size, &err_msg); + if (ret != 0) { + LOG(ERROR) << "key=" << std::string(key, key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values) { + // TODO: Implement GetWithPrefix - need to simplify Go wrapper interface first + // For now, return error as this requires complex memory management + LOG(ERROR) << "GetWithPrefix not yet implemented - requires Go wrapper interface simplification"; + return ErrorCode::INTERNAL_ERROR; +} + +ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key) { + char* err_msg = nullptr; + char* first_key_ptr = nullptr; + int first_key_size = 0; + int ret = EtcdStoreGetFirstKeyWithPrefixWrapper((char*)prefix, (int)prefix_size, + &first_key_ptr, &first_key_size, + &err_msg); + if (ret == -2) { + free(err_msg); + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + first_key = std::string(first_key_ptr, first_key_size); + free(first_key_ptr); + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size) { + char* err_msg = nullptr; + int ret = EtcdStoreDeleteRangeWrapper((char*)start_key, (int)start_key_size, + (char*)end_key, (int)end_key_size, + &err_msg); + if (ret != 0) { + LOG(ERROR) << "start_key=" << std::string(start_key, start_key_size) + << ", end_key=" << std::string(end_key, end_key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} #else ErrorCode EtcdHelper::ConnectToEtcdStoreClient( const std::string& etcd_endpoints) { @@ -200,6 +265,34 @@ ErrorCode EtcdHelper::CancelKeepAlive(EtcdLeaseId lease_id) { return ErrorCode::ETCD_OPERATION_ERROR; } +ErrorCode EtcdHelper::Put(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + #endif } // namespace mooncake \ No newline at end of file diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp new file mode 100644 index 0000000000..8a86cddf75 --- /dev/null +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -0,0 +1,221 @@ +#include "etcd_oplog_store.h" + +#include +#include + +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif + +#include "etcd_helper.h" + +namespace mooncake { + +EtcdOpLogStore::EtcdOpLogStore(const std::string& cluster_id) + : cluster_id_(cluster_id) {} + +ErrorCode EtcdOpLogStore::WriteOpLog(const OpLogEntry& entry) { + std::string key = BuildOpLogKey(entry.sequence_id); + std::string value = SerializeOpLogEntry(entry); + + ErrorCode err = EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), + value.size()); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to write OpLog entry, sequence_id=" + << entry.sequence_id; + return err; + } + + // Update latest sequence_id + err = UpdateLatestSequenceId(entry.sequence_id); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to update latest sequence_id, but OpLog entry " + "was written successfully"; + // Don't return error here, as the OpLog entry was written + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::ReadOpLog(uint64_t sequence_id, + OpLogEntry& entry) { + std::string key = BuildOpLogKey(sequence_id); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry, sequence_id=" + << sequence_id; + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::ReadOpLogSince(uint64_t start_sequence_id, + size_t limit, + std::vector& entries) { + // TODO: Implement ReadOpLogSince using GetWithPrefix + // For now, read entries one by one (inefficient but works) + entries.clear(); + entries.reserve(limit); + + uint64_t current_seq = start_sequence_id + 1; + for (size_t i = 0; i < limit; ++i) { + OpLogEntry entry; + ErrorCode err = ReadOpLog(current_seq, entry); + if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { + // No more entries + break; + } + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to read OpLog entry, sequence_id=" + << current_seq; + return err; + } + entries.push_back(entry); + current_seq++; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::GetLatestSequenceId(uint64_t& sequence_id) { + std::string key = BuildLatestKey(); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + try { + sequence_id = std::stoull(value); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse latest sequence_id: " << e.what(); + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::UpdateLatestSequenceId(uint64_t sequence_id) { + std::string key = BuildLatestKey(); + std::string value = std::to_string(sequence_id); + return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), value.size()); +} + +ErrorCode EtcdOpLogStore::RecordSnapshotSequenceId( + const std::string& snapshot_id, uint64_t sequence_id) { + std::string key = BuildSnapshotKey(snapshot_id); + std::string value = std::to_string(sequence_id); + return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), value.size()); +} + +ErrorCode EtcdOpLogStore::GetSnapshotSequenceId( + const std::string& snapshot_id, uint64_t& sequence_id) { + std::string key = BuildSnapshotKey(snapshot_id); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + try { + sequence_id = std::stoull(value); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse snapshot sequence_id: " << e.what(); + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::CleanupOpLogBefore(uint64_t before_sequence_id) { + // Build start and end keys for the range + std::string start_key = BuildOpLogKey(1); // Start from sequence_id 1 + std::string end_key = BuildOpLogKey(before_sequence_id); // End before this + + return EtcdHelper::DeleteRange(start_key.c_str(), start_key.size(), + end_key.c_str(), end_key.size()); +} + +std::string EtcdOpLogStore::BuildOpLogKey(uint64_t sequence_id) const { + std::ostringstream oss; + oss << kOpLogPrefix << cluster_id_ << "/" << sequence_id; + return oss.str(); +} + +std::string EtcdOpLogStore::BuildLatestKey() const { + std::ostringstream oss; + oss << kOpLogPrefix << cluster_id_ << kLatestSuffix; + return oss.str(); +} + +std::string EtcdOpLogStore::BuildSnapshotKey( + const std::string& snapshot_id) const { + std::ostringstream oss; + oss << kOpLogPrefix << cluster_id_ << kSnapshotSuffix << snapshot_id + << "/sequence_id"; + return oss.str(); +} + +std::string EtcdOpLogStore::SerializeOpLogEntry( + const OpLogEntry& entry) const { + Json::Value root; + root["sequence_id"] = static_cast(entry.sequence_id); + root["timestamp_ms"] = static_cast(entry.timestamp_ms); + root["op_type"] = static_cast(entry.op_type); + root["object_key"] = entry.object_key; + root["payload"] = entry.payload; + root["checksum"] = static_cast(entry.checksum); + root["prefix_hash"] = static_cast(entry.prefix_hash); + root["key_sequence_id"] = static_cast(entry.key_sequence_id); + + Json::StreamWriterBuilder builder; + builder["indentation"] = ""; // Compact format + std::unique_ptr writer(builder.newStreamWriter()); + std::ostringstream oss; + writer->write(root, &oss); + return oss.str(); +} + +bool EtcdOpLogStore::DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) const { + Json::Value root; + Json::CharReaderBuilder builder; + std::unique_ptr reader(builder.newCharReader()); + std::string errors; + + if (!reader->parse(json_str.data(), json_str.data() + json_str.size(), + &root, &errors)) { + LOG(ERROR) << "Failed to parse JSON: " << errors; + return false; + } + + try { + entry.sequence_id = root["sequence_id"].asUInt64(); + entry.timestamp_ms = root["timestamp_ms"].asUInt64(); + entry.op_type = static_cast(root["op_type"].asInt()); + entry.object_key = root["object_key"].asString(); + entry.payload = root["payload"].asString(); + entry.checksum = root["checksum"].asUInt(); + entry.prefix_hash = root["prefix_hash"].asUInt(); + entry.key_sequence_id = root["key_sequence_id"].asUInt64(); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to deserialize OpLogEntry: " << e.what(); + return false; + } + + return true; +} + +} // namespace mooncake + diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index c941e64bc1..984ea3b747 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -7,6 +7,8 @@ #include #include +#include "etcd_helper.h" +#include "etcd_oplog_store.h" #include "master_metric_manager.h" #include "segment.h" #include "types.h" @@ -73,6 +75,33 @@ MasterService::MasterService(const MasterServiceConfig& config) MasterMetricManager::instance().inc_total_file_capacity( global_file_segment_size_); } + + // Initialize EtcdOpLogStore if HA is enabled and etcd endpoints are configured + // Note: This requires STORE_USE_ETCD to be enabled at compile time +#ifdef STORE_USE_ETCD + if (enable_ha_ && !config.etcd_endpoints.empty() && !cluster_id_.empty()) { + ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(config.etcd_endpoints); + if (err == ErrorCode::OK) { + auto etcd_oplog_store = + std::make_shared(cluster_id_); + oplog_manager_.SetEtcdOpLogStore(etcd_oplog_store); + LOG(INFO) << "EtcdOpLogStore initialized for cluster_id=" + << cluster_id_; + } else { + LOG(WARNING) << "Failed to connect to etcd, OpLog will only be " + "stored in memory buffer"; + } + } else if (enable_ha_) { + LOG(WARNING) << "HA mode enabled but etcd endpoints or cluster_id not " + "configured, OpLog will only be stored in memory buffer"; + } +#else + if (enable_ha_) { + LOG(WARNING) << "HA mode enabled but STORE_USE_ETCD is not enabled at " + "compile time, OpLog will only be stored in memory buffer. " + "Recompile with -DSTORE_USE_ETCD=ON to enable etcd support."; + } +#endif } // Helper function to append OpLog entry diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index 7a12ef8229..623d3523b3 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -5,10 +5,18 @@ #include #include +#include "etcd_oplog_store.h" + namespace mooncake { OpLogManager::OpLogManager() = default; +void OpLogManager::SetEtcdOpLogStore( + std::shared_ptr etcd_oplog_store) { + std::unique_lock lock(mutex_); + etcd_oplog_store_ = etcd_oplog_store; +} + uint64_t OpLogManager::Append(OpType type, const std::string& key, const std::string& payload) { OpLogEntry entry; @@ -31,6 +39,21 @@ uint64_t OpLogManager::Append(OpType type, const std::string& key, } buffer_.emplace_back(std::move(entry)); + + // Write to etcd if EtcdOpLogStore is set + if (etcd_oplog_store_) { + // Release lock before writing to etcd to avoid blocking + lock.unlock(); + ErrorCode err = etcd_oplog_store_->WriteOpLog(entry); + if (err != ErrorCode::OK) { + // Log error but don't fail the operation + // The entry is already in the memory buffer + LOG(WARNING) << "Failed to write OpLog to etcd, sequence_id=" + << entry.sequence_id + << ", but entry is in memory buffer"; + } + } + return last_seq_id_; } From 7daf1237b4824065029595e867e74c1b0aa5348b Mon Sep 17 00:00:00 2001 From: BernardLee Date: Sat, 27 Dec 2025 09:11:20 +0800 Subject: [PATCH 11/59] fix --- mooncake-store/src/CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 8390fc9d59..f033e3cec6 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -16,8 +16,6 @@ set(MOONCAKE_STORE_SOURCES ha_helper.cpp segment.cpp transfer_task.cpp - etcd_helper.cpp - ha_helper.cpp rpc_service.cpp offset_allocator.cpp posix_file.cpp @@ -28,7 +26,7 @@ set(MOONCAKE_STORE_SOURCES file_storage.cpp oplog_manager.cpp etcd_oplog_store.cpp - // replication_service.cpp removed - using etcd-based OpLog sync instead + # replication_service.cpp removed - using etcd-based OpLog sync instead ) set(EXTRA_LIBS "") From f9f36601755e6235f271e723ee9a0976981085ad Mon Sep 17 00:00:00 2001 From: BernardLee Date: Sat, 27 Dec 2025 09:14:00 +0800 Subject: [PATCH 12/59] fix --- mooncake-store/src/oplog_manager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index 623d3523b3..255eebc563 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "etcd_oplog_store.h" From a95051ca4de2ca8fc4514ff0d480603bd138c82e Mon Sep 17 00:00:00 2001 From: BernardLee Date: Sat, 27 Dec 2025 09:43:57 +0800 Subject: [PATCH 13/59] fix --- mooncake-common/etcd/etcd_wrapper.go | 476 +++++++++++++++++---------- mooncake-store/include/etcd_helper.h | 27 ++ mooncake-store/src/etcd_helper.cpp | 47 +++ 3 files changed, 369 insertions(+), 181 deletions(-) diff --git a/mooncake-common/etcd/etcd_wrapper.go b/mooncake-common/etcd/etcd_wrapper.go index ea7aec51e9..ce49cef0ac 100644 --- a/mooncake-common/etcd/etcd_wrapper.go +++ b/mooncake-common/etcd/etcd_wrapper.go @@ -4,6 +4,9 @@ package main #include #include #include + +// Callback function type for Watch events +typedef void (*WatchCallbackFunc)(void* context, const char* key, size_t key_size, const char* value, size_t value_size, int event_type); */ import "C" @@ -21,18 +24,21 @@ import ( // and can be configured separately. var ( // etcd client for transform engine - globalClient *clientv3.Client - globalMutex sync.Mutex - globalRefCount int + globalClient *clientv3.Client + globalMutex sync.Mutex + globalRefCount int // etcd client for store - storeClient *clientv3.Client - storeMutex sync.Mutex + storeClient *clientv3.Client + storeMutex sync.Mutex // keep alive contexts for store - storeKeepAliveCtx = make(map[int64]context.CancelFunc) - storeKeepAliveMutex sync.Mutex + storeKeepAliveCtx = make(map[int64]context.CancelFunc) + storeKeepAliveMutex sync.Mutex // watch contexts for store - storeWatchCtx = make(map[string]context.CancelFunc) - storeWatchMutex sync.Mutex + storeWatchCtx = make(map[string]context.CancelFunc) + storeWatchMutex sync.Mutex + // watch contexts for prefix watch + storePrefixWatchCtx = make(map[string]context.CancelFunc) + storePrefixWatchMutex sync.Mutex ) //export NewEtcdClient @@ -44,30 +50,30 @@ func NewEtcdClient(endpoints *C.char, errMsg **C.char) int { return 0 } - MaxMsgSize := 32*1024*1024 - endpointStr := C.GoString(endpoints) - // Support multiple endpoints separated by comma or semicolon - // Normalize separators to semicolon first, then split - endpointStr = strings.ReplaceAll(endpointStr, ",", ";") - parts := strings.Split(endpointStr, ";") - var validEndpoints []string - for _, ep := range parts { - ep = strings.TrimSpace(ep) - if ep != "" { - validEndpoints = append(validEndpoints, ep) - } - } - if len(validEndpoints) == 0 { - *errMsg = C.CString("no valid endpoints provided") - return -1 - } - - cli, err := clientv3.New(clientv3.Config{ - Endpoints: validEndpoints, - DialTimeout: 5 * time.Second, - MaxCallSendMsgSize: MaxMsgSize, - MaxCallRecvMsgSize: MaxMsgSize, - }) + MaxMsgSize := 32 * 1024 * 1024 + endpointStr := C.GoString(endpoints) + // Support multiple endpoints separated by comma or semicolon + // Normalize separators to semicolon first, then split + endpointStr = strings.ReplaceAll(endpointStr, ",", ";") + parts := strings.Split(endpointStr, ";") + var validEndpoints []string + for _, ep := range parts { + ep = strings.TrimSpace(ep) + if ep != "" { + validEndpoints = append(validEndpoints, ep) + } + } + if len(validEndpoints) == 0 { + *errMsg = C.CString("no valid endpoints provided") + return -1 + } + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: validEndpoints, + DialTimeout: 5 * time.Second, + MaxCallSendMsgSize: MaxMsgSize, + MaxCallRecvMsgSize: MaxMsgSize, + }) if err != nil { *errMsg = C.CString(err.Error()) @@ -161,7 +167,7 @@ func NewStoreEtcdClient(endpoints *C.char, errMsg **C.char) int { endpointStr := C.GoString(endpoints) endpointList := strings.Split(endpointStr, ";") - + // Filter out any empty strings that might result from splitting var validEndpoints []string for _, ep := range endpointList { @@ -236,37 +242,37 @@ func EtcdStoreGrantLeaseWrapper(ttl int64, leaseId *int64, errMsg **C.char) int //export EtcdStoreCreateWithLeaseWrapper func EtcdStoreCreateWithLeaseWrapper(key *C.char, keySize C.int, value *C.char, valueSize C.int, leaseId int64, revisionId *int64, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - k := C.GoStringN(key, keySize) - v := C.GoStringN(value, valueSize) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - // Create a transaction - txn := storeClient.Txn(ctx) - - // Only put the key if it does not exist - resp, err := txn.If(clientv3.Compare(clientv3.CreateRevision(k), "=", 0)). - Then(clientv3.OpPut(k, v, clientv3.WithLease(clientv3.LeaseID(leaseId)))). - Commit() - - if err != nil { - *errMsg = C.CString(err.Error()) - return -1 - } - - // If the key already existed, resp.Succeeded will be false - // If we created the key, resp.Succeeded will be true - if resp.Succeeded { - *revisionId = resp.Header.Revision - return 0; - } else { - *errMsg = C.CString("etcd transaction failed") - return -2 - } + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + v := C.GoStringN(value, valueSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Create a transaction + txn := storeClient.Txn(ctx) + + // Only put the key if it does not exist + resp, err := txn.If(clientv3.Compare(clientv3.CreateRevision(k), "=", 0)). + Then(clientv3.OpPut(k, v, clientv3.WithLease(clientv3.LeaseID(leaseId)))). + Commit() + + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + // If the key already existed, resp.Succeeded will be false + // If we created the key, resp.Succeeded will be true + if resp.Succeeded { + *revisionId = resp.Header.Revision + return 0 + } else { + *errMsg = C.CString("etcd transaction failed") + return -2 + } } /* @@ -275,77 +281,77 @@ func EtcdStoreCreateWithLeaseWrapper(key *C.char, keySize C.int, value *C.char, * other than the one we want to delete. In that case, that context will * be deleted before being cancelled and will not be able to be cancelled * anymore. -*/ + */ func cancelAndDeleteWatch(k string) int { - storeWatchMutex.Lock() - defer storeWatchMutex.Unlock() - - if cancel, exists := storeWatchCtx[k]; exists { - cancel() - delete(storeWatchCtx, k) - return 0 - } + storeWatchMutex.Lock() + defer storeWatchMutex.Unlock() + + if cancel, exists := storeWatchCtx[k]; exists { + cancel() + delete(storeWatchCtx, k) + return 0 + } return -1 } //export EtcdStoreWatchUntilDeletedWrapper func EtcdStoreWatchUntilDeletedWrapper(key *C.char, keySize C.int, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - k := C.GoStringN(key, keySize) - - // Create a context with cancel function - ctx, cancel := context.WithCancel(context.Background()) - - // Store the cancel function - storeWatchMutex.Lock() - if _, exists := storeWatchCtx[k]; exists { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storeWatchMutex.Lock() + if _, exists := storeWatchCtx[k]; exists { storeWatchMutex.Unlock() - *errMsg = C.CString("This key is already being watched") - return -1 - } - storeWatchCtx[k] = cancel - storeWatchMutex.Unlock() + *errMsg = C.CString("This key is already being watched") + return -1 + } + storeWatchCtx[k] = cancel + storeWatchMutex.Unlock() // Make sure to delete from the map before returning defer cancelAndDeleteWatch(k) - // Start watching the key - watchChan := storeClient.Watch(ctx, k) - - // Wait for the key to be deleted - for { - select { - case watchResp, ok := <-watchChan: - if !ok { - // Channel closed unexpectedly - *errMsg = C.CString("watch channel closed unexpectedly") - return -1 - } - for _, event := range watchResp.Events { - if event.Type == clientv3.EventTypeDelete { - // Clean up the context when done - return 0 - } - } - case <-ctx.Done(): - // Context was cancelled + // Start watching the key + watchChan := storeClient.Watch(ctx, k) + + // Wait for the key to be deleted + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + // Channel closed unexpectedly + *errMsg = C.CString("watch channel closed unexpectedly") + return -1 + } + for _, event := range watchResp.Events { + if event.Type == clientv3.EventTypeDelete { + // Clean up the context when done + return 0 + } + } + case <-ctx.Done(): + // Context was cancelled *errMsg = C.CString("watch context cancelled") - return -2 - } - } + return -2 + } + } } //export EtcdStoreCancelWatchWrapper func EtcdStoreCancelWatchWrapper(key *C.char, keySize C.int, errMsg **C.char) int { - k := C.GoStringN(key, keySize) - if cancelAndDeleteWatch(k) == -1 { - *errMsg = C.CString("no watch context found for the given key") - return -1 - } - return 0 + k := C.GoStringN(key, keySize) + if cancelAndDeleteWatch(k) == -1 { + *errMsg = C.CString("no watch context found for the given key") + return -1 + } + return 0 } /* @@ -354,76 +360,76 @@ func EtcdStoreCancelWatchWrapper(key *C.char, keySize C.int, errMsg **C.char) in * other than the one we want to delete. In that case, that context will * be deleted before being cancelled and will not be able to be cancelled * anymore. -*/ + */ func cancelAndDeleteKeepAlive(leaseId int64) int { - storeKeepAliveMutex.Lock() - defer storeKeepAliveMutex.Unlock() - - if cancel, exists := storeKeepAliveCtx[leaseId]; exists { - cancel() - delete(storeKeepAliveCtx, leaseId) - return 0 - } + storeKeepAliveMutex.Lock() + defer storeKeepAliveMutex.Unlock() + + if cancel, exists := storeKeepAliveCtx[leaseId]; exists { + cancel() + delete(storeKeepAliveCtx, leaseId) + return 0 + } return -1 } //export EtcdStoreKeepAliveWrapper func EtcdStoreKeepAliveWrapper(leaseId int64, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - - // Create a context with cancel function - ctx, cancel := context.WithCancel(context.Background()) - - // Store the cancel function - storeKeepAliveMutex.Lock() + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storeKeepAliveMutex.Lock() if _, exists := storeKeepAliveCtx[leaseId]; exists { storeKeepAliveMutex.Unlock() - *errMsg = C.CString("This lease id is already being kept alive") - return -1 - } - storeKeepAliveCtx[leaseId] = cancel - storeKeepAliveMutex.Unlock() + *errMsg = C.CString("This lease id is already being kept alive") + return -1 + } + storeKeepAliveCtx[leaseId] = cancel + storeKeepAliveMutex.Unlock() // Make sure to delete from the map before returning - defer cancelAndDeleteKeepAlive(leaseId) - - // Start keep alive - keepAliveChan, err := storeClient.KeepAlive(ctx, clientv3.LeaseID(leaseId)) - if err != nil { - *errMsg = C.CString(err.Error()) - return -1 - } - - // Wait for keep alive responses - for { - select { - case resp, ok := <-keepAliveChan: - if !ok { - *errMsg = C.CString("keep alive channel closed") - return -1 - } - if resp == nil { - *errMsg = C.CString("keep alive response is nil") - return -1 - } - // Keep alive successful, continue - case <-ctx.Done(): + defer cancelAndDeleteKeepAlive(leaseId) + + // Start keep alive + keepAliveChan, err := storeClient.KeepAlive(ctx, clientv3.LeaseID(leaseId)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + // Wait for keep alive responses + for { + select { + case resp, ok := <-keepAliveChan: + if !ok { + *errMsg = C.CString("keep alive channel closed") + return -1 + } + if resp == nil { + *errMsg = C.CString("keep alive response is nil") + return -1 + } + // Keep alive successful, continue + case <-ctx.Done(): // Context cancelled *errMsg = C.CString("keep alive context cancelled") - return -2 - } - } + return -2 + } + } } //export EtcdStoreCancelKeepAliveWrapper func EtcdStoreCancelKeepAliveWrapper(leaseId int64, errMsg **C.char) int { - if cancelAndDeleteKeepAlive(leaseId) == -1 { - *errMsg = C.CString("no keep alive context found for the given lease ID") - return -1 - } - return 0 + if cancelAndDeleteKeepAlive(leaseId) == -1 { + *errMsg = C.CString("no keep alive context found for the given lease ID") + return -1 + } + return 0 } //export EtcdStorePutWrapper @@ -458,34 +464,34 @@ func EtcdStoreGetWithPrefixWrapper(prefix *C.char, prefixSize C.int, keys **C.ch *errMsg = C.CString(err.Error()) return -1 } - + if len(resp.Kvs) == 0 { *count = 0 return 0 } - + // Allocate arrays for keys and values keyCount := len(resp.Kvs) *count = C.int(keyCount) - + // Allocate memory for arrays keysArray := (*[1 << 30]*C.char)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof((*C.char)(nil))))) keySizesArray := (*[1 << 30]C.int)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof(C.int(0))))) valuesArray := (*[1 << 30]*C.char)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof((*C.char)(nil))))) valueSizesArray := (*[1 << 30]C.int)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof(C.int(0))))) - + for i, kv := range resp.Kvs { keysArray[i] = C.CString(string(kv.Key)) keySizesArray[i] = C.int(len(kv.Key)) valuesArray[i] = C.CString(string(kv.Value)) valueSizesArray[i] = C.int(len(kv.Value)) } - + *keys = (*C.char)(unsafe.Pointer(keysArray)) *keySizes = (*C.int)(unsafe.Pointer(keySizesArray)) *values = (*C.char)(unsafe.Pointer(valuesArray)) *valueSizes = (*C.int)(unsafe.Pointer(valueSizesArray)) - + return 0 } @@ -532,4 +538,112 @@ func EtcdStoreDeleteRangeWrapper(startKey *C.char, startKeySize C.int, endKey *C return 0 } +//export EtcdStoreWatchWithPrefixWrapper +func EtcdStoreWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, callbackContext unsafe.Pointer, callbackFunc C.WatchCallbackFunc, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + if callbackFunc == nil { + *errMsg = C.CString("callback function is nil") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storePrefixWatchMutex.Lock() + if _, exists := storePrefixWatchCtx[p]; exists { + storePrefixWatchMutex.Unlock() + *errMsg = C.CString("This prefix is already being watched") + return -1 + } + storePrefixWatchCtx[p] = cancel + storePrefixWatchMutex.Unlock() + + // Start watching in a goroutine + go func() { + defer cancelAndDeletePrefixWatch(p) + + // Start watching the prefix + watchChan := storeClient.Watch(ctx, p, clientv3.WithPrefix()) + + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + // Channel closed unexpectedly + return + } + if watchResp.Err() != nil { + // Watch error, stop watching + return + } + + // Process each event + for _, event := range watchResp.Events { + var keyPtr *C.char + var keySize C.size_t + var valuePtr *C.char + var valueSize C.size_t + var eventType C.int + + keyStr := string(event.Kv.Key) + keyPtr = C.CString(keyStr) + keySize = C.size_t(len(keyStr)) + + if event.Type == clientv3.EventTypePut { + eventType = C.int(0) // WatchEventTypePut + valueStr := string(event.Kv.Value) + valuePtr = C.CString(valueStr) + valueSize = C.size_t(len(valueStr)) + } else if event.Type == clientv3.EventTypeDelete { + eventType = C.int(1) // WatchEventTypeDelete + valuePtr = nil + valueSize = 0 + } + + // Call the C callback function + callbackFunc(callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType) + + // Free the C strings + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + } + case <-ctx.Done(): + // Context was cancelled + return + } + } + }() + + return 0 +} + +func cancelAndDeletePrefixWatch(p string) int { + storePrefixWatchMutex.Lock() + defer storePrefixWatchMutex.Unlock() + + if cancel, exists := storePrefixWatchCtx[p]; exists { + cancel() + delete(storePrefixWatchCtx, p) + return 0 + } + return -1 +} + +//export EtcdStoreCancelWatchWithPrefixWrapper +func EtcdStoreCancelWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, errMsg **C.char) int { + p := C.GoStringN(prefix, prefixSize) + if cancelAndDeletePrefixWatch(p) == -1 { + *errMsg = C.CString("no watch context found for the given prefix") + return -1 + } + return 0 +} + func main() {} diff --git a/mooncake-store/include/etcd_helper.h b/mooncake-store/include/etcd_helper.h index f4a2a51752..11e783ba54 100644 --- a/mooncake-store/include/etcd_helper.h +++ b/mooncake-store/include/etcd_helper.h @@ -139,6 +139,33 @@ class EtcdHelper { const char* end_key, const size_t end_key_size); + /* + * @brief Watch all keys with a given prefix for changes. + * This is a non-blocking function that starts watching in a background + * goroutine. Events are delivered via the callback function. + * @param prefix: The prefix to watch. + * @param prefix_size: The size of the prefix in bytes. + * @param callback_context: User context passed to the callback function. + * @param callback_func: Callback function called for each watch event. + * Signature: void callback(void* context, const char* key, size_t key_size, + * const char* value, size_t value_size, int event_type) + * event_type: 0 = PUT, 1 = DELETE + * @return: Error code. + */ + static ErrorCode WatchWithPrefix(const char* prefix, const size_t prefix_size, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, + const char*, size_t, int)); + + /* + * @brief Cancel watching a prefix. + * @param prefix: The prefix to stop watching. + * @param prefix_size: The size of the prefix in bytes. + * @return: Error code. + */ + static ErrorCode CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size); + private: // Variables that are used to ensure the etcd client // is only connected once. diff --git a/mooncake-store/src/etcd_helper.cpp b/mooncake-store/src/etcd_helper.cpp index fcee8043f3..4703b9a7ac 100644 --- a/mooncake-store/src/etcd_helper.cpp +++ b/mooncake-store/src/etcd_helper.cpp @@ -217,6 +217,39 @@ ErrorCode EtcdHelper::DeleteRange(const char* start_key, } return ErrorCode::OK; } + +ErrorCode EtcdHelper::WatchWithPrefix(const char* prefix, const size_t prefix_size, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, + const char*, size_t, int)) { + char* err_msg = nullptr; + // The callback function signature matches C.WatchCallbackFunc, + // so we can pass it directly (C++ function pointers are compatible with C function pointers) + int ret = EtcdStoreWatchWithPrefixWrapper((char*)prefix, (int)prefix_size, + callback_context, callback_func, + &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size) { + char* err_msg = nullptr; + int ret = EtcdStoreCancelWatchWithPrefixWrapper((char*)prefix, (int)prefix_size, + &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} #else ErrorCode EtcdHelper::ConnectToEtcdStoreClient( const std::string& etcd_endpoints) { @@ -293,6 +326,20 @@ ErrorCode EtcdHelper::DeleteRange(const char* start_key, return ErrorCode::ETCD_OPERATION_ERROR; } +ErrorCode EtcdHelper::WatchWithPrefix(const char* prefix, const size_t prefix_size, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, + const char*, size_t, int)) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + #endif } // namespace mooncake \ No newline at end of file From 693d5dea53cc39741579453abcd3aa7f2e08f849 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Sat, 27 Dec 2025 09:48:21 +0800 Subject: [PATCH 14/59] fix --- mooncake-common/etcd/etcd_wrapper.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/mooncake-common/etcd/etcd_wrapper.go b/mooncake-common/etcd/etcd_wrapper.go index ce49cef0ac..3d579c6d97 100644 --- a/mooncake-common/etcd/etcd_wrapper.go +++ b/mooncake-common/etcd/etcd_wrapper.go @@ -4,9 +4,6 @@ package main #include #include #include - -// Callback function type for Watch events -typedef void (*WatchCallbackFunc)(void* context, const char* key, size_t key_size, const char* value, size_t value_size, int event_type); */ import "C" @@ -529,17 +526,16 @@ func EtcdStoreDeleteRangeWrapper(startKey *C.char, startKeySize C.int, endKey *C end := C.GoStringN(endKey, endKeySize) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - resp, err := storeClient.Delete(ctx, start, clientv3.WithRange(end)) + _, err := storeClient.Delete(ctx, start, clientv3.WithRange(end)) if err != nil { *errMsg = C.CString(err.Error()) return -1 } - // resp.Deleted contains the number of deleted keys return 0 } //export EtcdStoreWatchWithPrefixWrapper -func EtcdStoreWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, callbackContext unsafe.Pointer, callbackFunc C.WatchCallbackFunc, errMsg **C.char) int { +func EtcdStoreWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, callbackContext unsafe.Pointer, callbackFunc unsafe.Pointer, errMsg **C.char) int { if storeClient == nil { *errMsg = C.CString("etcd client not initialized") return -1 @@ -584,16 +580,14 @@ func EtcdStoreWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, callbackC // Process each event for _, event := range watchResp.Events { - var keyPtr *C.char - var keySize C.size_t + keyStr := string(event.Kv.Key) + keyPtr := C.CString(keyStr) + keySize := C.size_t(len(keyStr)) + var valuePtr *C.char var valueSize C.size_t var eventType C.int - keyStr := string(event.Kv.Key) - keyPtr = C.CString(keyStr) - keySize = C.size_t(len(keyStr)) - if event.Type == clientv3.EventTypePut { eventType = C.int(0) // WatchEventTypePut valueStr := string(event.Kv.Value) @@ -606,7 +600,9 @@ func EtcdStoreWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, callbackC } // Call the C callback function - callbackFunc(callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType) + // Convert unsafe.Pointer to function pointer type and call it + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int))(callbackFunc) + (*callbackType)(callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType) // Free the C strings C.free(unsafe.Pointer(keyPtr)) From 59bf00e8ae378ab50c0981e9633efc890defd38e Mon Sep 17 00:00:00 2001 From: BernardLee Date: Sat, 27 Dec 2025 09:52:47 +0800 Subject: [PATCH 15/59] fix --- mooncake-store/src/master_service.cpp | 29 +++++++++++++-------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 984ea3b747..89b9d6d82a 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -76,24 +76,23 @@ MasterService::MasterService(const MasterServiceConfig& config) global_file_segment_size_); } - // Initialize EtcdOpLogStore if HA is enabled and etcd endpoints are configured + // Initialize EtcdOpLogStore if HA is enabled // Note: This requires STORE_USE_ETCD to be enabled at compile time + // Note: etcd connection should be established before MasterService construction + // (e.g., in MasterServiceSupervisor), so we can use the existing connection #ifdef STORE_USE_ETCD - if (enable_ha_ && !config.etcd_endpoints.empty() && !cluster_id_.empty()) { - ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(config.etcd_endpoints); - if (err == ErrorCode::OK) { - auto etcd_oplog_store = - std::make_shared(cluster_id_); - oplog_manager_.SetEtcdOpLogStore(etcd_oplog_store); - LOG(INFO) << "EtcdOpLogStore initialized for cluster_id=" - << cluster_id_; - } else { - LOG(WARNING) << "Failed to connect to etcd, OpLog will only be " - "stored in memory buffer"; - } + if (enable_ha_ && !cluster_id_.empty()) { + // Try to create EtcdOpLogStore - if etcd is not connected, operations will fail + // but we can still use memory buffer as fallback + auto etcd_oplog_store = + std::make_shared(cluster_id_); + oplog_manager_.SetEtcdOpLogStore(etcd_oplog_store); + LOG(INFO) << "EtcdOpLogStore initialized for cluster_id=" + << cluster_id_ << " (etcd connection should be established " + << "before MasterService construction)"; } else if (enable_ha_) { - LOG(WARNING) << "HA mode enabled but etcd endpoints or cluster_id not " - "configured, OpLog will only be stored in memory buffer"; + LOG(WARNING) << "HA mode enabled but cluster_id is empty, " + "OpLog will only be stored in memory buffer"; } #else if (enable_ha_) { From 78d38caa0220caa56684eabacd73c74c3cad9eee Mon Sep 17 00:00:00 2001 From: BernardLee Date: Sat, 27 Dec 2025 09:54:43 +0800 Subject: [PATCH 16/59] fix --- mooncake-store/src/etcd_helper.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mooncake-store/src/etcd_helper.cpp b/mooncake-store/src/etcd_helper.cpp index 4703b9a7ac..88c31d0fc2 100644 --- a/mooncake-store/src/etcd_helper.cpp +++ b/mooncake-store/src/etcd_helper.cpp @@ -219,14 +219,15 @@ ErrorCode EtcdHelper::DeleteRange(const char* start_key, } ErrorCode EtcdHelper::WatchWithPrefix(const char* prefix, const size_t prefix_size, - void* callback_context, - void (*callback_func)(void*, const char*, size_t, - const char*, size_t, int)) { + void* callback_context, + void (*callback_func)(void*, const char*, size_t, + const char*, size_t, int)) { char* err_msg = nullptr; - // The callback function signature matches C.WatchCallbackFunc, - // so we can pass it directly (C++ function pointers are compatible with C function pointers) + // Convert function pointer to void* for passing to Go function + // Note: This is safe because we're just passing the pointer, not calling it + void* callback_func_ptr = reinterpret_cast(callback_func); int ret = EtcdStoreWatchWithPrefixWrapper((char*)prefix, (int)prefix_size, - callback_context, callback_func, + callback_context, callback_func_ptr, &err_msg); if (ret != 0) { LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) From 0e572fe700c437f58a5b4afcd49655b1544b557a Mon Sep 17 00:00:00 2001 From: BernardLee Date: Sat, 27 Dec 2025 10:05:25 +0800 Subject: [PATCH 17/59] oplog watcher --- mooncake-store/include/oplog_watcher.h | 26 ++- mooncake-store/src/CMakeLists.txt | 1 + mooncake-store/src/oplog_watcher.cpp | 249 +++++++++++++++++++++++++ 3 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 mooncake-store/src/oplog_watcher.cpp diff --git a/mooncake-store/include/oplog_watcher.h b/mooncake-store/include/oplog_watcher.h index 9357464d81..3a53eb83a6 100644 --- a/mooncake-store/include/oplog_watcher.h +++ b/mooncake-store/include/oplog_watcher.h @@ -59,6 +59,18 @@ class OpLogWatcher { uint64_t GetLastProcessedSequenceId() const; private: + /** + * @brief Static callback function for etcd Watch + * @param context OpLogWatcher instance (passed as void*) + * @param key etcd key + * @param key_size key size + * @param value etcd value + * @param value_size value size + * @param event_type event type (0 = PUT, 1 = DELETE) + */ + static void WatchCallback(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, int event_type); + /** * @brief Watch etcd OpLog changes (runs in background thread) */ @@ -67,11 +79,19 @@ class OpLogWatcher { /** * @brief Process a Watch event * @param key etcd key - * @param value etcd value - * @param revision etcd revision + * @param value etcd value (JSON string for PUT events, empty for DELETE events) + * @param event_type Event type (0 = PUT, 1 = DELETE) */ void HandleWatchEvent(const std::string& key, const std::string& value, - int64_t revision); + int event_type); + + /** + * @brief Deserialize OpLogEntry from JSON string + * @param json_str JSON string + * @param entry Output OpLog entry + * @return true on success, false on failure + */ + bool DeserializeOpLogEntry(const std::string& json_str, OpLogEntry& entry); std::string etcd_endpoints_; std::string cluster_id_; diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index f033e3cec6..df4a8d8fac 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -26,6 +26,7 @@ set(MOONCAKE_STORE_SOURCES file_storage.cpp oplog_manager.cpp etcd_oplog_store.cpp + oplog_watcher.cpp # replication_service.cpp removed - using etcd-based OpLog sync instead ) diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp new file mode 100644 index 0000000000..08298b0f02 --- /dev/null +++ b/mooncake-store/src/oplog_watcher.cpp @@ -0,0 +1,249 @@ +#include "oplog_watcher.h" + +#include +#include +#include +#include + +#ifdef STORE_USE_ETCD +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "oplog_applier.h" + +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif + +namespace mooncake { + +OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), cluster_id_(cluster_id), applier_(applier) { + if (applier_ == nullptr) { + LOG(FATAL) << "OpLogApplier cannot be null"; + } +} + +OpLogWatcher::~OpLogWatcher() { + Stop(); +} + +void OpLogWatcher::Start() { + if (running_.load()) { + LOG(WARNING) << "OpLogWatcher is already running"; + return; + } + + running_.store(true); + watch_thread_ = std::thread(&OpLogWatcher::WatchOpLog, this); + LOG(INFO) << "OpLogWatcher started for cluster_id=" << cluster_id_; +} + +void OpLogWatcher::Stop() { + if (!running_.load()) { + return; + } + + running_.store(false); + +#ifdef STORE_USE_ETCD + // Cancel the watch + std::string watch_prefix = "/oplog/" + cluster_id_ + "/"; + ErrorCode err = EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to cancel watch for prefix " << watch_prefix + << ", error=" << static_cast(err); + } +#endif + + // Wait for watch thread to finish + if (watch_thread_.joinable()) { + watch_thread_.join(); + } + + LOG(INFO) << "OpLogWatcher stopped"; +} + +bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries) { +#ifdef STORE_USE_ETCD + EtcdOpLogStore oplog_store(cluster_id_); + ErrorCode err = oplog_store.ReadOpLogSince(start_seq_id, 1000, entries); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to read OpLog since sequence_id=" << start_seq_id + << ", error=" << static_cast(err); + return false; + } + LOG(INFO) << "Read " << entries.size() << " OpLog entries since sequence_id=" + << start_seq_id; + return true; +#else + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot read OpLog from etcd"; + return false; +#endif +} + +uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { + return last_processed_sequence_id_.load(); +} + +// Static callback function for etcd Watch (defined before WatchOpLog uses it) +void OpLogWatcher::WatchCallback(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, int event_type) { + OpLogWatcher* watcher = static_cast(context); + if (watcher == nullptr) { + LOG(ERROR) << "OpLogWatcher context is null"; + return; + } + + std::string key_str(key, key_size); + std::string value_str; + if (value != nullptr && value_size > 0) { + value_str = std::string(value, value_size); + } + + watcher->HandleWatchEvent(key_str, value_str, event_type); +} + +void OpLogWatcher::WatchOpLog() { +#ifdef STORE_USE_ETCD + LOG(INFO) << "OpLog watch thread started for cluster_id=" << cluster_id_; + + std::string watch_prefix = "/oplog/" + cluster_id_ + "/"; + + // Start watching - pass static callback function and this pointer as context + ErrorCode err = EtcdHelper::WatchWithPrefix( + watch_prefix.c_str(), watch_prefix.size(), this, WatchCallback); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start watch for prefix " << watch_prefix + << ", error=" << static_cast(err); + running_.store(false); + return; + } + + LOG(INFO) << "Watch started for prefix " << watch_prefix; + + // The watch is now running in the background (via Go goroutine) + // We just need to keep the thread alive until Stop() is called + while (running_.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + LOG(INFO) << "OpLog watch thread stopped"; +#else + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot watch OpLog from etcd"; + running_.store(false); +#endif +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type) { + // event_type: 0 = PUT, 1 = DELETE + if (event_type == 1) { + // DELETE event - OpLog entry was cleaned up + VLOG(1) << "OpLog entry deleted: " << key; + return; + } + + if (event_type != 0) { + LOG(WARNING) << "Unknown event type: " << event_type << " for key: " << key; + return; + } + + // Skip the "latest" key and snapshot keys + if (key.find("/latest") != std::string::npos || + key.find("/snapshot/") != std::string::npos) { + return; + } + + // Parse the OpLog entry from JSON + OpLogEntry entry; + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry from key: " << key; + return; + } + + // Apply the OpLog entry + if (applier_->ApplyOpLogEntry(entry)) { + last_processed_sequence_id_.store(entry.sequence_id); + VLOG(2) << "Applied OpLog entry: sequence_id=" << entry.sequence_id + << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + } else { + LOG(WARNING) << "Failed to apply OpLog entry: sequence_id=" + << entry.sequence_id; + } +} + +bool OpLogWatcher::DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) { + Json::Value root; + Json::CharReaderBuilder reader; + std::string errs; + std::istringstream s(json_str); + + if (!Json::parseFromStream(reader, s, &root, &errs)) { + LOG(ERROR) << "Failed to parse OpLogEntry JSON: " << errs; + return false; + } + + entry.sequence_id = root.get("sequence_id", 0).asUInt64(); + entry.timestamp_ms = root.get("timestamp_ms", 0).asUInt64(); + entry.op_type = static_cast(root.get("op_type", 0).asInt()); + entry.object_key = root.get("object_key", "").asString(); + entry.payload = root.get("payload", "").asString(); + entry.checksum = root.get("checksum", 0).asUInt(); + entry.prefix_hash = root.get("prefix_hash", 0).asUInt(); + entry.key_sequence_id = root.get("key_sequence_id", 0).asUInt64(); + return true; +} + +} // namespace mooncake + +#else // STORE_USE_ETCD not defined + +namespace mooncake { + +OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), cluster_id_(cluster_id), applier_(applier) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +OpLogWatcher::~OpLogWatcher() { + Stop(); +} + +void OpLogWatcher::Start() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::Stop() { + // No-op when STORE_USE_ETCD is not enabled +} + +bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { + return last_processed_sequence_id_.load(); +} + +void OpLogWatcher::WatchOpLog() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +} // namespace mooncake + +#endif // STORE_USE_ETCD + From e082439ad19512f8755847206f8527586fed46ca Mon Sep 17 00:00:00 2001 From: BernardLee Date: Sat, 27 Dec 2025 10:59:08 +0800 Subject: [PATCH 18/59] ApplyOpLogEntry --- mooncake-store/include/metadata_store.h | 48 +++++ mooncake-store/include/oplog_applier.h | 4 +- mooncake-store/src/CMakeLists.txt | 1 + mooncake-store/src/oplog_applier.cpp | 250 ++++++++++++++++++++++++ 4 files changed, 300 insertions(+), 3 deletions(-) create mode 100644 mooncake-store/include/metadata_store.h create mode 100644 mooncake-store/src/oplog_applier.cpp diff --git a/mooncake-store/include/metadata_store.h b/mooncake-store/include/metadata_store.h new file mode 100644 index 0000000000..3451bd248e --- /dev/null +++ b/mooncake-store/include/metadata_store.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include + +namespace mooncake { + +/** + * @brief Abstract interface for metadata storage on Standby + * + * This interface provides basic operations for storing and managing object metadata. + * In a full implementation, this would mirror MasterService's metadata_shards_ structure. + */ +class MetadataStore { + public: + virtual ~MetadataStore() = default; + + /** + * @brief Put or update metadata for a key + * @param key Object key + * @param payload Optional payload data (JSON serialized metadata) + * @return true on success, false on failure + */ + virtual bool Put(const std::string& key, const std::string& payload = std::string()) = 0; + + /** + * @brief Remove metadata for a key + * @param key Object key + * @return true if key was found and removed, false otherwise + */ + virtual bool Remove(const std::string& key) = 0; + + /** + * @brief Check if a key exists + * @param key Object key + * @return true if key exists, false otherwise + */ + virtual bool Exists(const std::string& key) const = 0; + + /** + * @brief Get the count of keys in the store + * @return Number of keys + */ + virtual size_t GetKeyCount() const = 0; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/oplog_applier.h b/mooncake-store/include/oplog_applier.h index 23fd2383a7..67fdad1f25 100644 --- a/mooncake-store/include/oplog_applier.h +++ b/mooncake-store/include/oplog_applier.h @@ -8,12 +8,10 @@ #include #include "oplog_manager.h" +#include "metadata_store.h" namespace mooncake { -// Forward declaration -class MetadataStore; - /** * @brief Apply OpLog entries to Standby metadata store with ordering guarantee * diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index df4a8d8fac..08c0aa64fa 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -27,6 +27,7 @@ set(MOONCAKE_STORE_SOURCES oplog_manager.cpp etcd_oplog_store.cpp oplog_watcher.cpp + oplog_applier.cpp # replication_service.cpp removed - using etcd-based OpLog sync instead ) diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp new file mode 100644 index 0000000000..e7884351c5 --- /dev/null +++ b/mooncake-store/src/oplog_applier.cpp @@ -0,0 +1,250 @@ +#include "oplog_applier.h" + +#include + +#include +#include + +#include "metadata_store.h" + +namespace mooncake { + +OpLogApplier::OpLogApplier(MetadataStore* metadata_store) + : metadata_store_(metadata_store), expected_sequence_id_(1) { + if (metadata_store_ == nullptr) { + LOG(FATAL) << "OpLogApplier: metadata_store cannot be null"; + } +} + +bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { + // Check sequence order + if (!CheckSequenceOrder(entry)) { + // Order violation - add to pending entries + std::lock_guard lock(pending_mutex_); + pending_entries_[entry.sequence_id] = entry; + LOG(WARNING) << "OpLogApplier: sequence order violation, sequence_id=" + << entry.sequence_id << ", expected=" << expected_sequence_id_ + << ", key=" << entry.object_key + << ", added to pending entries"; + return false; + } + + // Apply the operation based on type + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + default: + LOG(ERROR) << "OpLogApplier: unsupported op_type=" + << static_cast(entry.op_type) + << ", sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key; + return false; + } + + // Update expected sequence ID + expected_sequence_id_ = entry.sequence_id + 1; + + // Update key sequence ID + { + std::lock_guard lock(key_sequence_mutex_); + key_sequence_map_[entry.object_key] = entry.key_sequence_id; + } + + // Try to process pending entries + ProcessPendingEntries(); + + return true; +} + +size_t OpLogApplier::ApplyOpLogEntries(const std::vector& entries) { + size_t applied_count = 0; + for (const auto& entry : entries) { + if (ApplyOpLogEntry(entry)) { + applied_count++; + } + } + return applied_count; +} + +uint64_t OpLogApplier::GetKeySequenceId(const std::string& key) const { + std::lock_guard lock(key_sequence_mutex_); + auto it = key_sequence_map_.find(key); + if (it != key_sequence_map_.end()) { + return it->second; + } + return 0; +} + +uint64_t OpLogApplier::GetExpectedSequenceId() const { + return expected_sequence_id_; +} + +void OpLogApplier::Recover(uint64_t last_applied_sequence_id) { + expected_sequence_id_ = last_applied_sequence_id + 1; + LOG(INFO) << "OpLogApplier: recovered from sequence_id=" + << last_applied_sequence_id + << ", expected_sequence_id set to=" << expected_sequence_id_; +} + +size_t OpLogApplier::ProcessPendingEntries() { + std::lock_guard lock(pending_mutex_); + size_t processed_count = 0; + + // Process entries in order + while (!pending_entries_.empty()) { + auto it = pending_entries_.begin(); + const OpLogEntry& entry = it->second; + + // Check if this entry can be applied now + if (entry.sequence_id == expected_sequence_id_) { + // Release lock before applying (to avoid deadlock) + OpLogEntry entry_copy = entry; + pending_entries_.erase(it); + + // Apply the entry (this will update expected_sequence_id_) + // We need to call ApplyOpLogEntry but without sequence check + // Since we've already verified the sequence_id matches expected_sequence_id_, + // we can directly apply it. + + // Actually, we should just apply it normally, but we need to skip + // the CheckSequenceOrder since we've already verified it. + // For now, let's just call ApplyOpLogEntry again, which will work + // because expected_sequence_id_ should now match. + // But this is inefficient. Let's do it properly: + + // Apply based on type + switch (entry_copy.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry_copy); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry_copy); + break; + case OpType::REMOVE: + ApplyRemove(entry_copy); + break; + default: + LOG(ERROR) << "OpLogApplier: unsupported op_type in pending entry"; + continue; + } + + // Update expected sequence ID + expected_sequence_id_ = entry_copy.sequence_id + 1; + + // Update key sequence ID + { + std::lock_guard key_lock(key_sequence_mutex_); + key_sequence_map_[entry_copy.object_key] = entry_copy.key_sequence_id; + } + + processed_count++; + } else { + // Cannot process more entries yet + break; + } + } + + if (processed_count > 0) { + LOG(INFO) << "OpLogApplier: processed " << processed_count + << " pending entries, expected_sequence_id now=" + << expected_sequence_id_; + } + + return processed_count; +} + +bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { + // Check global sequence order + if (entry.sequence_id != expected_sequence_id_) { + return false; + } + + // Check per-key sequence order + { + std::lock_guard lock(key_sequence_mutex_); + auto it = key_sequence_map_.find(entry.object_key); + if (it != key_sequence_map_.end()) { + // Key exists - check that new key_sequence_id is greater + if (entry.key_sequence_id <= it->second) { + LOG(WARNING) << "OpLogApplier: key sequence order violation, key=" + << entry.object_key + << ", new key_sequence_id=" << entry.key_sequence_id + << ", current key_sequence_id=" << it->second; + return false; + } + } + // If key doesn't exist, any key_sequence_id is valid (should be >= 1) + if (entry.key_sequence_id < 1) { + LOG(WARNING) << "OpLogApplier: invalid key_sequence_id=" + << entry.key_sequence_id << " for new key=" + << entry.object_key; + return false; + } + } + + return true; +} + +void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) { + // For now, payload is empty in current implementation. + // In the future, payload may contain serialized metadata. + // For now, we just mark the key as existing. + if (!metadata_store_->Put(entry.object_key, entry.payload)) { + LOG(ERROR) << "OpLogApplier: failed to Put key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } else { + VLOG(1) << "OpLogApplier: applied PUT_END, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } +} + +void OpLogApplier::ApplyPutRevoke(const OpLogEntry& entry) { + // PUT_REVOKE means the object should be removed from metadata store + // (but the key itself may still exist if there are other replicas) + // For now, we treat it as a remove operation + // In the future, we may need to handle partial replica removal + if (!metadata_store_->Remove(entry.object_key)) { + LOG(WARNING) << "OpLogApplier: failed to Remove key=" << entry.object_key + << " in PUT_REVOKE, sequence_id=" << entry.sequence_id + << " (key may not exist)"; + } else { + VLOG(1) << "OpLogApplier: applied PUT_REVOKE, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } +} + +void OpLogApplier::ApplyRemove(const OpLogEntry& entry) { + if (!metadata_store_->Remove(entry.object_key)) { + LOG(WARNING) << "OpLogApplier: failed to Remove key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << " (key may not exist)"; + } else { + VLOG(1) << "OpLogApplier: applied REMOVE, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } +} + +bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) { + // TODO: Implement request missing OpLog from etcd + // This will be implemented in Phase 3 + LOG(WARNING) << "OpLogApplier: RequestMissingOpLog not yet implemented, " + << "missing_seq_id=" << missing_seq_id; + return false; +} + +void OpLogApplier::ScheduleWaitForMissingEntries(uint64_t missing_seq_id) { + // TODO: Implement scheduling wait for missing entries + // This will be implemented in Phase 3 + LOG(WARNING) << "OpLogApplier: ScheduleWaitForMissingEntries not yet implemented, " + << "missing_seq_id=" << missing_seq_id; +} + +} // namespace mooncake + From c8227bc9cca263d9a5c18baa78cef13585791cfc Mon Sep 17 00:00:00 2001 From: BernardLee Date: Sat, 27 Dec 2025 11:07:59 +0800 Subject: [PATCH 19/59] HotStandbyService --- mooncake-store/include/hot_standby_service.h | 42 +++-- mooncake-store/src/hot_standby_service.cpp | 169 +++++++++++++++---- 2 files changed, 164 insertions(+), 47 deletions(-) diff --git a/mooncake-store/include/hot_standby_service.h b/mooncake-store/include/hot_standby_service.h index 5ba88c5007..439947cd71 100644 --- a/mooncake-store/include/hot_standby_service.h +++ b/mooncake-store/include/hot_standby_service.h @@ -7,8 +7,12 @@ #include #include #include +#include +#include "metadata_store.h" +#include "oplog_applier.h" #include "oplog_manager.h" +#include "oplog_watcher.h" #include "types.h" namespace mooncake { @@ -60,10 +64,14 @@ class HotStandbyService { /** * @brief Start connecting to Primary and begin replication - * @param primary_address Address of the Primary Master + * @param primary_address Address of the Primary Master (not used with etcd-based sync) + * @param etcd_endpoints Comma-separated etcd endpoints + * @param cluster_id Cluster identifier for OpLog path * @return ErrorCode::OK on success */ - ErrorCode Start(const std::string& primary_address); + ErrorCode Start(const std::string& primary_address, + const std::string& etcd_endpoints, + const std::string& cluster_id); /** * @brief Stop replication and disconnect from Primary @@ -112,6 +120,7 @@ class HotStandbyService { /** * @brief Apply a single OpLog entry to local metadata store * @param entry The OpLog entry to apply + * @deprecated Use OpLogApplier instead */ void ApplyOpLogEntry(const OpLogEntry& entry); @@ -134,15 +143,28 @@ class HotStandbyService { HotStandbyConfig config_; - // Metadata store (simplified - in full implementation this would be - // a complete replica of MasterService's metadata) - // For now, we use a placeholder structure - struct MetadataStore { - // Placeholder: In full implementation, this would mirror - // MasterService's metadata_shards_ structure - size_t entry_count{0}; + // Simple in-memory metadata store implementation + class StandbyMetadataStore : public MetadataStore { + public: + bool Put(const std::string& key, + const std::string& payload = std::string()) override; + bool Remove(const std::string& key) override; + bool Exists(const std::string& key) const override; + size_t GetKeyCount() const override; + + private: + mutable std::mutex mutex_; + std::unordered_map store_; }; - std::unique_ptr metadata_store_; + std::unique_ptr metadata_store_; + + // OpLog replication components + std::unique_ptr oplog_applier_; + std::unique_ptr oplog_watcher_; + + // Configuration for etcd-based OpLog sync + std::string etcd_endpoints_; + std::string cluster_id_; // Replication state std::shared_ptr replication_stream_; diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index ab07d99205..7ac3e8725a 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -5,21 +5,58 @@ #include #include +#include "etcd_helper.h" +#include "etcd_oplog_store.h" #include "master_service.h" +#include "oplog_applier.h" #include "oplog_manager.h" +#include "oplog_watcher.h" namespace mooncake { HotStandbyService::HotStandbyService(const HotStandbyConfig& config) : config_(config) { - metadata_store_ = std::make_unique(); + metadata_store_ = std::make_unique(); + oplog_applier_ = + std::make_unique(metadata_store_.get()); +} + +// StandbyMetadataStore implementation +bool HotStandbyService::StandbyMetadataStore::Put(const std::string& key, + const std::string& payload) { + std::lock_guard lock(mutex_); + store_[key] = payload; // payload may be empty for now + return true; +} + +bool HotStandbyService::StandbyMetadataStore::Remove(const std::string& key) { + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it != store_.end()) { + store_.erase(it); + return true; + } + return false; +} + +bool HotStandbyService::StandbyMetadataStore::Exists( + const std::string& key) const { + std::lock_guard lock(mutex_); + return store_.find(key) != store_.end(); +} + +size_t HotStandbyService::StandbyMetadataStore::GetKeyCount() const { + std::lock_guard lock(mutex_); + return store_.size(); } HotStandbyService::~HotStandbyService() { Stop(); } -ErrorCode HotStandbyService::Start(const std::string& primary_address) { +ErrorCode HotStandbyService::Start(const std::string& primary_address, + const std::string& etcd_endpoints, + const std::string& cluster_id) { std::lock_guard lock(mutex_); if (running_.load()) { @@ -28,7 +65,46 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address) { } config_.primary_address = primary_address; + etcd_endpoints_ = etcd_endpoints; + cluster_id_ = cluster_id; + +#ifdef STORE_USE_ETCD + // Connect to etcd + ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(etcd_endpoints.c_str()); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to connect to etcd: " << etcd_endpoints; + return err; + } + + // Create OpLogWatcher + oplog_watcher_ = std::make_unique( + etcd_endpoints, cluster_id, oplog_applier_.get()); + running_.store(true); + is_connected_.store(true); + + // Read historical OpLog entries first + // Get the last applied sequence ID from OpLogApplier + uint64_t last_applied_seq_id = oplog_applier_->GetExpectedSequenceId() - 1; + if (last_applied_seq_id == 0) { + // First time - start from sequence_id 0 (will read from sequence_id 1) + last_applied_seq_id = 0; + } + + std::vector historical_entries; + if (oplog_watcher_->ReadOpLogSince(last_applied_seq_id, historical_entries)) { + LOG(INFO) << "Read " << historical_entries.size() + << " historical OpLog entries, applying..."; + // Apply historical entries + size_t applied_count = oplog_applier_->ApplyOpLogEntries(historical_entries); + LOG(INFO) << "Applied " << applied_count + << " historical OpLog entries"; + } else { + LOG(WARNING) << "Failed to read historical OpLog entries, continuing anyway"; + } + + // Start OpLogWatcher (this will start watching etcd in background) + oplog_watcher_->Start(); // Start background threads replication_thread_ = std::thread(&HotStandbyService::ReplicationLoop, this); @@ -37,9 +113,13 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address) { std::thread(&HotStandbyService::VerificationLoop, this); } - LOG(INFO) << "HotStandbyService started, connecting to Primary: " - << primary_address; + LOG(INFO) << "HotStandbyService started, watching etcd OpLog for cluster: " + << cluster_id; return ErrorCode::OK; +#else + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot start HotStandbyService"; + return ErrorCode::INTERNAL_ERROR; +#endif } void HotStandbyService::Stop() { @@ -48,7 +128,13 @@ void HotStandbyService::Stop() { } running_.store(false); - DisconnectFromPrimary(); + is_connected_.store(false); + + // Stop OpLogWatcher + if (oplog_watcher_) { + oplog_watcher_->Stop(); + oplog_watcher_.reset(); + } // Wait for threads to finish if (replication_thread_.joinable()) { @@ -63,7 +149,20 @@ void HotStandbyService::Stop() { StandbySyncStatus HotStandbyService::GetSyncStatus() const { StandbySyncStatus status; - status.applied_seq_id = applied_seq_id_.load(); + + // Get applied sequence ID from OpLogApplier + if (oplog_applier_) { + status.applied_seq_id = oplog_applier_->GetExpectedSequenceId() - 1; + if (status.applied_seq_id == 0) { + status.applied_seq_id = applied_seq_id_.load(); // Fallback + } + } else { + status.applied_seq_id = applied_seq_id_.load(); + } + + // Get primary sequence ID from etcd (if OpLogWatcher is available) + // For now, we use a placeholder - in full implementation we would + // query etcd for the latest sequence_id status.primary_seq_id = primary_seq_id_.load(); status.is_connected = is_connected_.load(); @@ -121,36 +220,36 @@ std::unique_ptr HotStandbyService::Promote() { size_t HotStandbyService::GetMetadataCount() const { std::lock_guard lock(mutex_); - return metadata_store_ ? metadata_store_->entry_count : 0; + return metadata_store_ ? metadata_store_->GetKeyCount() : 0; } void HotStandbyService::ReplicationLoop() { - LOG(INFO) << "Replication loop started"; + LOG(INFO) << "Replication loop started (etcd-based OpLog sync)"; + + // With etcd-based OpLog sync, OpLogWatcher handles the actual watching + // in its own thread. This loop now just monitors the status and updates + // metrics. while (running_.load()) { - // Try to connect if not connected if (!is_connected_.load()) { - if (ConnectToPrimary()) { - is_connected_.store(true); - LOG(INFO) << "Connected to Primary: " << config_.primary_address; - } else { - // Retry after a delay - std::this_thread::sleep_for(std::chrono::seconds(1)); - continue; - } + // Not connected - wait a bit before checking again + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; } - // In full implementation, this would: - // 1. Receive OpLog entries from Primary via gRPC stream - // 2. Process them in batches - // 3. Apply to local metadata store + // Update applied_seq_id from OpLogApplier + if (oplog_applier_) { + uint64_t current_applied = oplog_applier_->GetExpectedSequenceId() - 1; + if (current_applied > 0) { + applied_seq_id_.store(current_applied); + } + } - // For now, this is a placeholder that simulates receiving entries - // In the actual implementation, this would block on the gRPC stream - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + // TODO: Update primary_seq_id by querying etcd for latest sequence_id + // For now, we assume it's being updated elsewhere - // Placeholder: Simulate receiving entries - // TODO: Replace with actual gRPC stream reading + // Sleep and check again + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } LOG(INFO) << "Replication loop stopped"; @@ -222,23 +321,19 @@ void HotStandbyService::ProcessOpLogBatch( } bool HotStandbyService::ConnectToPrimary() { - // In full implementation, this would: - // 1. Create a gRPC channel to Primary - // 2. Establish a bidirectional stream for OpLog replication - // 3. Send initial sync request with current applied_seq_id - // 4. Start receiving OpLog entries - - // For now, this is a placeholder - LOG(INFO) << "Connecting to Primary: " << config_.primary_address - << " (placeholder)"; - return false; // Return false to indicate not yet implemented + // With etcd-based OpLog sync, connection is handled by OpLogWatcher + // This method is kept for compatibility but is no longer used + LOG(INFO) << "ConnectToPrimary called (no-op with etcd-based sync)"; + return true; } void HotStandbyService::DisconnectFromPrimary() { + // With etcd-based OpLog sync, disconnection is handled by OpLogWatcher + // This method is kept for compatibility if (is_connected_.load()) { is_connected_.store(false); replication_stream_.reset(); - LOG(INFO) << "Disconnected from Primary"; + LOG(INFO) << "Disconnected from Primary (etcd-based sync)"; } } From 83eb05a41c00ef29eea4c5ecf50c2e6968096f9c Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 11:32:18 +0800 Subject: [PATCH 20/59] fix --- mooncake-store/include/ha_helper.h | 22 +++++- mooncake-store/src/ha_helper.cpp | 110 ++++++++++++++++++++++++++++- 2 files changed, 128 insertions(+), 4 deletions(-) diff --git a/mooncake-store/include/ha_helper.h b/mooncake-store/include/ha_helper.h index 897ba53a5c..f6774d6a93 100644 --- a/mooncake-store/include/ha_helper.h +++ b/mooncake-store/include/ha_helper.h @@ -3,12 +3,15 @@ #include +#include +#include #include #include #include -#include "types.h" +#include "hot_standby_service.h" #include "master_config.h" +#include "types.h" namespace mooncake { @@ -77,10 +80,27 @@ class MasterServiceSupervisor { ~MasterServiceSupervisor(); private: + /** + * @brief Start HotStandbyService when there is an existing leader + * @param mv_helper MasterViewHelper instance + * @param current_leader Current leader address + */ + void StartStandbyService(MasterViewHelper& mv_helper, + const std::string& current_leader); + + /** + * @brief Stop HotStandbyService + */ + void StopStandbyService(); + // coro_rpc server thread std::thread server_thread_; MasterServiceSupervisorConfig config_; + + // HotStandbyService for standby mode + std::unique_ptr standby_service_; + std::atomic standby_running_{false}; }; } // namespace mooncake diff --git a/mooncake-store/src/ha_helper.cpp b/mooncake-store/src/ha_helper.cpp index 07906772af..a7f79e9708 100644 --- a/mooncake-store/src/ha_helper.cpp +++ b/mooncake-store/src/ha_helper.cpp @@ -1,5 +1,12 @@ #include "ha_helper.h" + +#include + +#include +#include + #include "etcd_helper.h" +#include "hot_standby_service.h" #include "rpc_service.h" namespace mooncake { @@ -131,11 +138,59 @@ int MasterServiceSupervisor::Start() { << config_.etcd_endpoints; return -1; } - LOG(INFO) << "Trying to elect self as leader..."; + +#ifdef STORE_USE_ETCD + // Connect to etcd for OpLog sync + if (EtcdHelper::ConnectToEtcdStoreClient(config_.etcd_endpoints.c_str()) != + ErrorCode::OK) { + LOG(ERROR) << "Failed to connect to etcd store client: " + << config_.etcd_endpoints; + return -1; + } +#endif + + LOG(INFO) << "Checking for existing leader..."; EtcdLeaseId lease_id = 0; - // view_version will be updated by ElectLeader and then used in - // WrappedMasterService ViewVersionId view_version = 0; + + // Check if there is already a leader + std::string current_leader; + ViewVersionId current_version = 0; + auto ret = mv_helper.GetMasterView(current_leader, current_version); + + if (ret == ErrorCode::OK) { + // There is an existing leader, start Standby service + LOG(INFO) << "Found existing leader: " << current_leader + << ", starting Standby service..."; + StartStandbyService(mv_helper, current_leader); + + // Build master_view_key (same logic as MasterViewHelper) + std::string cluster_id = config_.cluster_id; + if (!cluster_id.empty() && cluster_id.back() != '/') { + cluster_id += '/'; + } + std::string master_view_key = "mooncake-store/" + cluster_id + "master_view"; + + // Watch until leader is deleted + LOG(INFO) << "Watching for leadership change..."; + auto watch_ret = EtcdHelper::WatchUntilDeleted( + master_view_key.c_str(), master_view_key.size()); + + // Stop Standby service when leader disappears + StopStandbyService(); + + if (watch_ret != ErrorCode::OK) { + LOG(ERROR) << "Error watching for leadership change: " << watch_ret; + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + LOG(INFO) << "Leader disappeared, trying to elect self as leader..."; + } else { + LOG(INFO) << "No existing leader found, trying to elect self as leader..."; + } + + // Try to elect self as leader mv_helper.ElectLeader(config_.local_hostname, view_version, lease_id); // Start a thread to keep the leader alive @@ -186,7 +241,56 @@ int MasterServiceSupervisor::Start() { return 0; } +void MasterServiceSupervisor::StartStandbyService(MasterViewHelper& mv_helper, + const std::string& current_leader) { +#ifdef STORE_USE_ETCD + if (standby_running_.load()) { + LOG(WARNING) << "Standby service is already running"; + return; + } + + HotStandbyConfig standby_config; + standby_config.standby_id = config_.local_hostname; + standby_config.primary_address = current_leader; + standby_config.verification_interval_sec = 30; + standby_config.max_replication_lag_entries = 1000; + standby_config.enable_verification = false; // Disable verification for now + + standby_service_ = std::make_unique(standby_config); + + ErrorCode err = standby_service_->Start( + current_leader, config_.etcd_endpoints, config_.cluster_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start Standby service: " << err; + standby_service_.reset(); + return; + } + + standby_running_.store(true); + LOG(INFO) << "Standby service started successfully"; +#else + LOG(WARNING) << "STORE_USE_ETCD is not enabled, cannot start Standby service"; +#endif +} + +void MasterServiceSupervisor::StopStandbyService() { +#ifdef STORE_USE_ETCD + if (!standby_running_.load()) { + return; + } + + if (standby_service_) { + standby_service_->Stop(); + standby_service_.reset(); + } + + standby_running_.store(false); + LOG(INFO) << "Standby service stopped"; +#endif +} + MasterServiceSupervisor::~MasterServiceSupervisor() { + StopStandbyService(); if (server_thread_.joinable()) { server_thread_.join(); } From 44d0d74bf6eea7f724823ed4033b4dab6906c5da Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 11:38:57 +0800 Subject: [PATCH 21/59] fix --- mooncake-store/src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 08c0aa64fa..8f726fd780 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -28,6 +28,7 @@ set(MOONCAKE_STORE_SOURCES etcd_oplog_store.cpp oplog_watcher.cpp oplog_applier.cpp + hot_standby_service.cpp # replication_service.cpp removed - using etcd-based OpLog sync instead ) From fd893ae71f2bc402731c8e9c384cec2fcc9fabfa Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 11:43:32 +0800 Subject: [PATCH 22/59] fix --- mooncake-store/src/hot_standby_service.cpp | 39 ++++++---------------- 1 file changed, 10 insertions(+), 29 deletions(-) diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index 7ac3e8725a..646f9e65e4 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -280,37 +280,18 @@ void HotStandbyService::VerificationLoop() { } void HotStandbyService::ApplyOpLogEntry(const OpLogEntry& entry) { - // In full implementation, this would apply the OpLog entry to - // the local metadata store, mirroring the operations in MasterService. - - // For now, this is a placeholder that just updates counters - switch (entry.op_type) { - case OpType::PUT_END: - // Create or update metadata for the key - if (metadata_store_) { - metadata_store_->entry_count++; - } - break; - case OpType::PUT_REVOKE: - case OpType::REMOVE: - // Remove metadata for the key - if (metadata_store_ && metadata_store_->entry_count > 0) { - metadata_store_->entry_count--; - } - break; - case OpType::LEASE_RENEW: - // LEASE_RENEW is no longer used. Standby does not perform eviction, - // so it doesn't need to track lease renewals. DELETE events from - // Primary will handle object removal. - // This case is kept for backward compatibility with old OpLog entries. - break; - default: - LOG(WARNING) << "Unknown OpType: " - << static_cast(entry.op_type); - break; - } + // NOTE: This method is deprecated. OpLog entries are now applied via + // OpLogApplier, which is called by OpLogWatcher. This method is kept + // for backward compatibility but should not be used in the new etcd-based + // implementation. + // Update applied_seq_id for status tracking applied_seq_id_.store(entry.sequence_id); + + // The actual application is handled by OpLogApplier via OpLogWatcher + VLOG(2) << "ApplyOpLogEntry called (deprecated), sequence_id=" + << entry.sequence_id << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; } void HotStandbyService::ProcessOpLogBatch( From dccb000327a2da870d636ee2968465365445ec88 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 11:55:50 +0800 Subject: [PATCH 23/59] mod --- doc/zh/rfc-oplog-implementation-plan.md | 22 ++++++----- mooncake-store/src/oplog_applier.cpp | 49 +++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/doc/zh/rfc-oplog-implementation-plan.md b/doc/zh/rfc-oplog-implementation-plan.md index 7e1907e23a..114de5a9c7 100644 --- a/doc/zh/rfc-oplog-implementation-plan.md +++ b/doc/zh/rfc-oplog-implementation-plan.md @@ -220,22 +220,26 @@ - 等待一段时间后，从 etcd 读取缺失的条目 - 序列号连续后，按顺序应用 -#### 3.2 实现回滚和重放机制（3-4 天） +#### 3.2 实现 key 级别乱序处理（1-2 天） **文件**： - `mooncake-store/src/oplog_applier.cpp`（修改） **功能**： -- [ ] `RollbackAndReplayKey()`：回滚并重放指定 key -- [ ] `ReadOpLogForKey()`：从 etcd 读取指定 key 的所有 OpLog -- [ ] 维护 `key_first_sequence_id_` 记录首次 sequence_id -- [ ] 使用 `keys_under_rollback_` 防止并发回滚 -- [ ] 异步执行回滚，不阻塞正常处理 +- [x] 检测到 key 级别乱序时，直接删除该 key 的 metadata +- [x] 从 `key_sequence_map_` 中删除该 key +- [x] 删除后继续处理当前 OpLog 条目（如果全局序列号正确） + +**设计说明**： +- 简化方案：不进行回滚和重放，因为前面的数据可能已经丢失 +- 当检测到 `key_sequence_id` 乱序时，直接删除该 key +- 如果后续有 PUT_END 操作，会重新创建该 key +- 这样避免了数据不一致的风险，实现更简单可靠 **验收标准**： -- 检测到 key 级别乱序时，触发回滚和重放 -- 回滚期间，新的 OpLog 暂时跳过 -- 回滚完成后，metadata 正确 +- 检测到 key 级别乱序时，正确删除该 key 的 metadata +- 删除后可以继续处理后续的 OpLog 条目 +- 不会导致数据不一致 #### 3.3 实现错误处理和恢复（2-3 天） diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index e7884351c5..0796fe6e85 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -18,11 +18,54 @@ OpLogApplier::OpLogApplier(MetadataStore* metadata_store) bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { // Check sequence order - if (!CheckSequenceOrder(entry)) { - // Order violation - add to pending entries + bool order_valid = CheckSequenceOrder(entry); + + // If key-level sequence order violation detected, delete the key + if (!order_valid) { + // Check if it's a key-level violation (not just global sequence violation) + bool is_key_violation = false; + uint64_t current_key_seq_id = 0; + { + std::lock_guard lock(key_sequence_mutex_); + auto it = key_sequence_map_.find(entry.object_key); + if (it != key_sequence_map_.end()) { + current_key_seq_id = it->second; + // Key exists and key_sequence_id is not greater + // Also check that global sequence is correct (only handle key-level violations) + if (entry.sequence_id == expected_sequence_id_ && + entry.key_sequence_id <= current_key_seq_id) { + is_key_violation = true; + } + } + } + + if (is_key_violation) { + // Key-level sequence violation: delete the key and its metadata + LOG(WARNING) << "OpLogApplier: key sequence order violation detected, " + << "deleting key=" << entry.object_key + << ", new key_sequence_id=" << entry.key_sequence_id + << ", current key_sequence_id=" << current_key_seq_id; + + // Delete from metadata store + metadata_store_->Remove(entry.object_key); + + // Delete from key_sequence_map_ + { + std::lock_guard lock(key_sequence_mutex_); + key_sequence_map_.erase(entry.object_key); + } + + // Now the key is deleted, we can continue to process this entry + // since global sequence is correct (we checked above) + order_valid = true; + } + } + + if (!order_valid) { + // Global sequence violation - add to pending entries std::lock_guard lock(pending_mutex_); pending_entries_[entry.sequence_id] = entry; - LOG(WARNING) << "OpLogApplier: sequence order violation, sequence_id=" + LOG(WARNING) << "OpLogApplier: global sequence order violation, sequence_id=" << entry.sequence_id << ", expected=" << expected_sequence_id_ << ", key=" << entry.object_key << ", added to pending entries"; From 6ba449b580ac9a808a9b14b7dfbbe13bd51cd761 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 14:47:16 +0800 Subject: [PATCH 24/59] oplog seq --- mooncake-store/include/oplog_applier.h | 28 +++- mooncake-store/src/hot_standby_service.cpp | 8 +- mooncake-store/src/oplog_applier.cpp | 153 ++++++++++++++++++--- 3 files changed, 164 insertions(+), 25 deletions(-) diff --git a/mooncake-store/include/oplog_applier.h b/mooncake-store/include/oplog_applier.h index 67fdad1f25..302c77a699 100644 --- a/mooncake-store/include/oplog_applier.h +++ b/mooncake-store/include/oplog_applier.h @@ -1,7 +1,9 @@ #pragma once +#include #include #include +#include #include #include #include @@ -12,6 +14,9 @@ namespace mooncake { +// Forward declaration +class EtcdOpLogStore; + /** * @brief Apply OpLog entries to Standby metadata store with ordering guarantee * @@ -23,8 +28,10 @@ class OpLogApplier { /** * @brief Constructor * @param metadata_store Metadata store to apply changes to + * @param cluster_id Cluster ID for accessing etcd OpLog (optional, for requesting missing OpLog) */ - explicit OpLogApplier(MetadataStore* metadata_store); + explicit OpLogApplier(MetadataStore* metadata_store, + const std::string& cluster_id = std::string()); /** * @brief Apply a single OpLog entry (with ordering checks) @@ -106,6 +113,17 @@ class OpLogApplier { MetadataStore* metadata_store_; + // EtcdOpLogStore for requesting missing OpLog entries (optional) + std::string cluster_id_; + mutable std::mutex etcd_oplog_store_mutex_; + mutable std::unique_ptr etcd_oplog_store_; + + /** + * @brief Get or create EtcdOpLogStore instance (lazy initialization) + * @return Pointer to EtcdOpLogStore, or nullptr if cluster_id is not set + */ + EtcdOpLogStore* GetEtcdOpLogStore() const; + // Track per-key sequence ID for ordering guarantee mutable std::mutex key_sequence_mutex_; std::unordered_map key_sequence_map_; @@ -113,7 +131,15 @@ class OpLogApplier { // Track pending entries (entries with non-continuous sequence IDs) mutable std::mutex pending_mutex_; std::map pending_entries_; + + // Track missing sequence IDs that we're waiting for + std::map missing_sequence_ids_; + uint64_t expected_sequence_id_{1}; + + // Constants for missing entry handling + static constexpr int kMissingEntryWaitSeconds = 5; // Wait 5 seconds before requesting + static constexpr int kMaxPendingEntries = 1000; // Max pending entries before giving up }; } // namespace mooncake diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index 646f9e65e4..a7c0cab85d 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -17,8 +17,9 @@ namespace mooncake { HotStandbyService::HotStandbyService(const HotStandbyConfig& config) : config_(config) { metadata_store_ = std::make_unique(); - oplog_applier_ = - std::make_unique(metadata_store_.get()); + // OpLogApplier will be created in Start() with cluster_id + // For now, create without cluster_id (will be updated in Start) + oplog_applier_ = std::make_unique(metadata_store_.get()); } // StandbyMetadataStore implementation @@ -76,6 +77,9 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, return err; } + // Recreate OpLogApplier with cluster_id (for requesting missing OpLog) + oplog_applier_ = std::make_unique(metadata_store_.get(), cluster_id); + // Create OpLogWatcher oplog_watcher_ = std::make_unique( etcd_endpoints, cluster_id, oplog_applier_.get()); diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index 0796fe6e85..b3c9bf49d1 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -5,17 +5,37 @@ #include #include +#include "etcd_oplog_store.h" #include "metadata_store.h" namespace mooncake { -OpLogApplier::OpLogApplier(MetadataStore* metadata_store) - : metadata_store_(metadata_store), expected_sequence_id_(1) { +OpLogApplier::OpLogApplier(MetadataStore* metadata_store, + const std::string& cluster_id) + : metadata_store_(metadata_store), + cluster_id_(cluster_id), + expected_sequence_id_(1) { if (metadata_store_ == nullptr) { LOG(FATAL) << "OpLogApplier: metadata_store cannot be null"; } } +EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const { +#ifdef STORE_USE_ETCD + if (cluster_id_.empty()) { + return nullptr; + } + + std::lock_guard lock(etcd_oplog_store_mutex_); + if (!etcd_oplog_store_) { + etcd_oplog_store_ = std::make_unique(cluster_id_); + } + return etcd_oplog_store_.get(); +#else + return nullptr; +#endif +} + bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { // Check sequence order bool order_valid = CheckSequenceOrder(entry); @@ -64,11 +84,20 @@ bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { if (!order_valid) { // Global sequence violation - add to pending entries std::lock_guard lock(pending_mutex_); + + // Check if we've exceeded max pending entries + if (pending_entries_.size() >= static_cast(kMaxPendingEntries)) { + LOG(ERROR) << "OpLogApplier: too many pending entries (" + << pending_entries_.size() << "), discarding entry sequence_id=" + << entry.sequence_id << ", key=" << entry.object_key; + return false; + } + pending_entries_[entry.sequence_id] = entry; LOG(WARNING) << "OpLogApplier: global sequence order violation, sequence_id=" << entry.sequence_id << ", expected=" << expected_sequence_id_ << ", key=" << entry.object_key - << ", added to pending entries"; + << ", added to pending entries (total: " << pending_entries_.size() << ")"; return false; } @@ -137,6 +166,46 @@ void OpLogApplier::Recover(uint64_t last_applied_sequence_id) { } size_t OpLogApplier::ProcessPendingEntries() { + // Check for missing sequence IDs and request them if needed (before acquiring lock) + uint64_t missing_seq_to_request = 0; + { + std::lock_guard lock(pending_mutex_); + if (!pending_entries_.empty()) { + uint64_t first_pending_seq = pending_entries_.begin()->first; + if (first_pending_seq > expected_sequence_id_) { + // There's a gap - we're missing entries between expected_sequence_id_ and first_pending_seq + uint64_t missing_seq = expected_sequence_id_; + auto missing_it = missing_sequence_ids_.find(missing_seq); + auto now = std::chrono::steady_clock::now(); + + if (missing_it == missing_sequence_ids_.end()) { + // First time we see this missing sequence - record the time + missing_sequence_ids_[missing_seq] = now; + ScheduleWaitForMissingEntries(missing_seq); + } else { + // Check if we've waited long enough + auto wait_duration = std::chrono::duration_cast( + now - missing_it->second); + if (wait_duration.count() >= kMissingEntryWaitSeconds) { + // Mark for requesting (we'll do it after releasing the lock) + missing_seq_to_request = missing_seq; + } + } + } + } + } + + // Request missing OpLog if needed (outside the lock to avoid deadlock) + bool retrieved_missing = false; + if (missing_seq_to_request > 0) { + retrieved_missing = RequestMissingOpLog(missing_seq_to_request); + if (retrieved_missing) { + std::lock_guard lock(pending_mutex_); + missing_sequence_ids_.erase(missing_seq_to_request); + } + } + + // Now process pending entries std::lock_guard lock(pending_mutex_); size_t processed_count = 0; @@ -151,17 +220,6 @@ size_t OpLogApplier::ProcessPendingEntries() { OpLogEntry entry_copy = entry; pending_entries_.erase(it); - // Apply the entry (this will update expected_sequence_id_) - // We need to call ApplyOpLogEntry but without sequence check - // Since we've already verified the sequence_id matches expected_sequence_id_, - // we can directly apply it. - - // Actually, we should just apply it normally, but we need to skip - // the CheckSequenceOrder since we've already verified it. - // For now, let's just call ApplyOpLogEntry again, which will work - // because expected_sequence_id_ should now match. - // But this is inefficient. Let's do it properly: - // Apply based on type switch (entry_copy.op_type) { case OpType::PUT_END: @@ -187,6 +245,9 @@ size_t OpLogApplier::ProcessPendingEntries() { key_sequence_map_[entry_copy.object_key] = entry_copy.key_sequence_id; } + // Remove from missing list if it was there + missing_sequence_ids_.erase(entry_copy.sequence_id); + processed_count++; } else { // Cannot process more entries yet @@ -194,6 +255,20 @@ size_t OpLogApplier::ProcessPendingEntries() { } } + // Clean up old missing sequence IDs (older than 1 minute) + auto now = std::chrono::steady_clock::now(); + for (auto it = missing_sequence_ids_.begin(); it != missing_sequence_ids_.end();) { + auto age = std::chrono::duration_cast(now - it->second); + if (age.count() > 60) { + // Too old, remove it + LOG(WARNING) << "OpLogApplier: giving up on missing sequence_id=" + << it->first << " after " << age.count() << " seconds"; + it = missing_sequence_ids_.erase(it); + } else { + ++it; + } + } + if (processed_count > 0) { LOG(INFO) << "OpLogApplier: processed " << processed_count << " pending entries, expected_sequence_id now=" @@ -275,18 +350,52 @@ void OpLogApplier::ApplyRemove(const OpLogEntry& entry) { } bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) { - // TODO: Implement request missing OpLog from etcd - // This will be implemented in Phase 3 - LOG(WARNING) << "OpLogApplier: RequestMissingOpLog not yet implemented, " - << "missing_seq_id=" << missing_seq_id; +#ifdef STORE_USE_ETCD + EtcdOpLogStore* oplog_store = GetEtcdOpLogStore(); + if (oplog_store == nullptr) { + LOG(WARNING) << "OpLogApplier: cannot request missing OpLog, cluster_id not set"; + return false; + } + + OpLogEntry entry; + ErrorCode err = oplog_store->ReadOpLog(missing_seq_id, entry); + if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { + LOG(INFO) << "OpLogApplier: missing OpLog entry not found in etcd, sequence_id=" + << missing_seq_id; + return false; + } + if (err != ErrorCode::OK) { + LOG(ERROR) << "OpLogApplier: failed to read missing OpLog from etcd, sequence_id=" + << missing_seq_id << ", error=" << static_cast(err); + return false; + } + + // Successfully retrieved the missing OpLog entry + LOG(INFO) << "OpLogApplier: retrieved missing OpLog entry, sequence_id=" + << missing_seq_id << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + + // Add to pending entries + // Note: We don't call ProcessPendingEntries() here to avoid potential recursion. + // The caller (ProcessPendingEntries itself) will process the entry in the next loop. + { + std::lock_guard lock(pending_mutex_); + pending_entries_[entry.sequence_id] = entry; + } + + return true; +#else + LOG(WARNING) << "OpLogApplier: STORE_USE_ETCD not enabled, cannot request missing OpLog"; return false; +#endif } void OpLogApplier::ScheduleWaitForMissingEntries(uint64_t missing_seq_id) { - // TODO: Implement scheduling wait for missing entries - // This will be implemented in Phase 3 - LOG(WARNING) << "OpLogApplier: ScheduleWaitForMissingEntries not yet implemented, " - << "missing_seq_id=" << missing_seq_id; + // This method is called when we first detect a missing sequence_id. + // The actual waiting and requesting is handled in ProcessPendingEntries(). + // We just log it here for tracking. + VLOG(1) << "OpLogApplier: scheduling wait for missing sequence_id=" << missing_seq_id + << ", will request after " << kMissingEntryWaitSeconds << " seconds"; } } // namespace mooncake From 373f82312946e943329e4ba9f840e22f978cf537 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 15:06:29 +0800 Subject: [PATCH 25/59] fix --- doc/zh/rfc-oplog-implementation-plan.md | 20 +++- mooncake-store/include/oplog_watcher.h | 22 ++++ mooncake-store/src/oplog_watcher.cpp | 134 +++++++++++++++++++++--- 3 files changed, 154 insertions(+), 22 deletions(-) diff --git a/doc/zh/rfc-oplog-implementation-plan.md b/doc/zh/rfc-oplog-implementation-plan.md index 114de5a9c7..da1cb40dc8 100644 --- a/doc/zh/rfc-oplog-implementation-plan.md +++ b/doc/zh/rfc-oplog-implementation-plan.md @@ -246,17 +246,27 @@ **文件**： - `mooncake-store/src/oplog_applier.cpp`（修改） - `mooncake-store/src/oplog_watcher.cpp`（修改） +- `mooncake-store/include/oplog_watcher.h`（修改） **功能**： -- [ ] Watch 断开时自动重连 -- [ ] 从 etcd 读取失败时的重试机制 -- [ ] 应用 OpLog 失败时的错误处理 -- [ ] 记录乱序频率，超过阈值时触发全量同步 +- [x] Watch 断开时自动重连（指数退避策略） +- [x] 重连时同步遗漏的 OpLog 条目（`SyncMissedEntries()`） +- [x] 连续错误计数，超过阈值（10次）时触发重连 +- [x] 重连成功后重置错误计数 +- [x] 完善的日志记录 + +**实现细节**： +- `kMaxConsecutiveErrors = 10`：连续错误超过此阈值触发重连 +- `kReconnectDelayMs = 1000`：初始重连延迟（毫秒） +- `kMaxReconnectDelayMs = 30000`：最大重连延迟（30秒） +- `TryReconnect()`：指数退避重连，重连前同步遗漏条目 +- `SyncMissedEntries()`：从 etcd 读取 `last_processed_sequence_id_` 之后的条目 **验收标准**： - Watch 断开后可以自动重连 +- 重连期间遗漏的 OpLog 可以被正确同步 - 错误处理完善，不会导致服务崩溃 -- 有完善的日志和监控 +- 有完善的日志记录 ### Phase 3 里程碑 diff --git a/mooncake-store/include/oplog_watcher.h b/mooncake-store/include/oplog_watcher.h index 3a53eb83a6..35a9f68e00 100644 --- a/mooncake-store/include/oplog_watcher.h +++ b/mooncake-store/include/oplog_watcher.h @@ -93,12 +93,34 @@ class OpLogWatcher { */ bool DeserializeOpLogEntry(const std::string& json_str, OpLogEntry& entry); + /** + * @brief Attempt to reconnect after watch failure + */ + void TryReconnect(); + + /** + * @brief Sync missed OpLog entries after reconnection + * @return true if sync was successful + */ + bool SyncMissedEntries(); + std::string etcd_endpoints_; std::string cluster_id_; OpLogApplier* applier_; std::atomic running_{false}; std::thread watch_thread_; std::atomic last_processed_sequence_id_{0}; + + // Error handling and recovery + std::atomic consecutive_errors_{0}; + std::atomic reconnect_count_{0}; + std::atomic watch_healthy_{false}; + + // Constants for error handling + static constexpr int kMaxConsecutiveErrors = 10; + static constexpr int kReconnectDelayMs = 1000; + static constexpr int kMaxReconnectDelayMs = 30000; + static constexpr int kSyncBatchSize = 1000; }; } // namespace mooncake diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp index 08298b0f02..f5dbd43556 100644 --- a/mooncake-store/src/oplog_watcher.cpp +++ b/mooncake-store/src/oplog_watcher.cpp @@ -1,5 +1,6 @@ #include "oplog_watcher.h" +#include #include #include #include @@ -113,22 +114,44 @@ void OpLogWatcher::WatchOpLog() { std::string watch_prefix = "/oplog/" + cluster_id_ + "/"; - // Start watching - pass static callback function and this pointer as context - ErrorCode err = EtcdHelper::WatchWithPrefix( - watch_prefix.c_str(), watch_prefix.size(), this, WatchCallback); - if (err != ErrorCode::OK) { - LOG(ERROR) << "Failed to start watch for prefix " << watch_prefix - << ", error=" << static_cast(err); - running_.store(false); - return; - } - - LOG(INFO) << "Watch started for prefix " << watch_prefix; - - // The watch is now running in the background (via Go goroutine) - // We just need to keep the thread alive until Stop() is called while (running_.load()) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + // Start watching - pass static callback function and this pointer as context + ErrorCode err = EtcdHelper::WatchWithPrefix( + watch_prefix.c_str(), watch_prefix.size(), this, WatchCallback); + + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start watch for prefix " << watch_prefix + << ", error=" << static_cast(err); + watch_healthy_.store(false); + + // Try to reconnect + TryReconnect(); + continue; + } + + LOG(INFO) << "Watch started for prefix " << watch_prefix; + watch_healthy_.store(true); + consecutive_errors_.store(0); + + // The watch is now running in the background (via Go goroutine) + // We just need to keep the thread alive until Stop() is called or watch fails + while (running_.load() && watch_healthy_.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Periodically check watch health + if (consecutive_errors_.load() >= kMaxConsecutiveErrors) { + LOG(WARNING) << "Too many consecutive errors (" << consecutive_errors_.load() + << "), reconnecting watch..."; + watch_healthy_.store(false); + break; + } + } + + if (running_.load() && !watch_healthy_.load()) { + // Cancel current watch before reconnecting + EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + TryReconnect(); + } } LOG(INFO) << "OpLog watch thread stopped"; @@ -138,17 +161,80 @@ void OpLogWatcher::WatchOpLog() { #endif } +void OpLogWatcher::TryReconnect() { + if (!running_.load()) { + return; + } + + int reconnect_attempt = reconnect_count_.fetch_add(1) + 1; + + // Calculate delay with exponential backoff + int delay_ms = std::min(kReconnectDelayMs * reconnect_attempt, kMaxReconnectDelayMs); + + LOG(INFO) << "Attempting to reconnect watch (attempt #" << reconnect_attempt + << "), waiting " << delay_ms << "ms..."; + + std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); + + // Sync any missed entries before resuming watch + if (SyncMissedEntries()) { + LOG(INFO) << "Successfully synced missed OpLog entries"; + } else { + LOG(WARNING) << "Failed to sync missed OpLog entries, continuing anyway"; + } +} + +bool OpLogWatcher::SyncMissedEntries() { +#ifdef STORE_USE_ETCD + uint64_t last_seq = last_processed_sequence_id_.load(); + if (last_seq == 0) { + // No entries processed yet, nothing to sync + return true; + } + + LOG(INFO) << "Syncing missed OpLog entries since sequence_id=" << last_seq; + + std::vector entries; + if (!ReadOpLogSince(last_seq, entries)) { + LOG(ERROR) << "Failed to read missed OpLog entries"; + return false; + } + + if (entries.empty()) { + LOG(INFO) << "No missed OpLog entries to sync"; + return true; + } + + LOG(INFO) << "Syncing " << entries.size() << " missed OpLog entries"; + + for (const auto& entry : entries) { + if (applier_->ApplyOpLogEntry(entry)) { + last_processed_sequence_id_.store(entry.sequence_id); + } else { + LOG(WARNING) << "Failed to apply missed OpLog entry, sequence_id=" + << entry.sequence_id; + } + } + + return true; +#else + return false; +#endif +} + void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, int event_type) { // event_type: 0 = PUT, 1 = DELETE if (event_type == 1) { // DELETE event - OpLog entry was cleaned up VLOG(1) << "OpLog entry deleted: " << key; + consecutive_errors_.store(0); // Watch is working return; } if (event_type != 0) { LOG(WARNING) << "Unknown event type: " << event_type << " for key: " << key; + consecutive_errors_.fetch_add(1); return; } @@ -162,18 +248,23 @@ void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& v OpLogEntry entry; if (!DeserializeOpLogEntry(value, entry)) { LOG(ERROR) << "Failed to deserialize OpLog entry from key: " << key; + consecutive_errors_.fetch_add(1); return; } // Apply the OpLog entry if (applier_->ApplyOpLogEntry(entry)) { last_processed_sequence_id_.store(entry.sequence_id); + consecutive_errors_.store(0); // Reset error counter on success + reconnect_count_.store(0); // Reset reconnect counter on success VLOG(2) << "Applied OpLog entry: sequence_id=" << entry.sequence_id << ", op_type=" << static_cast(entry.op_type) << ", key=" << entry.object_key; } else { - LOG(WARNING) << "Failed to apply OpLog entry: sequence_id=" - << entry.sequence_id; + // ApplyOpLogEntry returns false for out-of-order entries, + // which is expected behavior, not an error + VLOG(1) << "OpLog entry not applied (may be out of order): sequence_id=" + << entry.sequence_id; } } @@ -243,6 +334,15 @@ void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& v LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; } +void OpLogWatcher::TryReconnect() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +bool OpLogWatcher::SyncMissedEntries() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + } // namespace mooncake #endif // STORE_USE_ETCD From 4b73166ea8fafa5930e02d0f8af5d4a5b469ae4a Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 15:13:18 +0800 Subject: [PATCH 26/59] fix doc --- doc/zh/rfc-oplog-implementation-plan.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/zh/rfc-oplog-implementation-plan.md b/doc/zh/rfc-oplog-implementation-plan.md index da1cb40dc8..f96d17c754 100644 --- a/doc/zh/rfc-oplog-implementation-plan.md +++ b/doc/zh/rfc-oplog-implementation-plan.md @@ -283,7 +283,9 @@ --- -## Phase 4：快照集成和清理（优先级：P2） +## Phase 4：快照集成和清理（优先级：P2）⏸️ 暂缓 + +> **状态**：暂缓，等待与快照团队协调讨论后再实现。 ### 目标 集成快照机制，实现 OpLog 清理，减少 etcd 存储压力。 From fdb90d4838dfff2c00e0d85d45337802eb93320a Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 16:04:53 +0800 Subject: [PATCH 27/59] delete key_seq_map_ --- mooncake-store/include/oplog_applier.h | 11 ++- mooncake-store/include/oplog_manager.h | 7 +- mooncake-store/src/oplog_applier.cpp | 107 +++---------------------- mooncake-store/src/oplog_manager.cpp | 9 ++- 4 files changed, 27 insertions(+), 107 deletions(-) diff --git a/mooncake-store/include/oplog_applier.h b/mooncake-store/include/oplog_applier.h index 302c77a699..2e611bbe26 100644 --- a/mooncake-store/include/oplog_applier.h +++ b/mooncake-store/include/oplog_applier.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #include "oplog_manager.h" @@ -48,9 +47,10 @@ class OpLogApplier { size_t ApplyOpLogEntries(const std::vector& entries); /** - * @brief Get the current sequence ID for a key + * @brief Get the current sequence ID for a key (DEPRECATED) * @param key Object key - * @return Current sequence ID, or 0 if key not found + * @return Always returns 0 - key_sequence_id is no longer tracked + * @deprecated Use global sequence_id for ordering */ uint64_t GetKeySequenceId(const std::string& key) const; @@ -124,9 +124,8 @@ class OpLogApplier { */ EtcdOpLogStore* GetEtcdOpLogStore() const; - // Track per-key sequence ID for ordering guarantee - mutable std::mutex key_sequence_mutex_; - std::unordered_map key_sequence_map_; + // Note: key_sequence_map_ has been removed. + // Global sequence_id is sufficient for ordering guarantee. // Track pending entries (entries with non-continuous sequence IDs) mutable std::mutex pending_mutex_; diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index 3f680ba130..ab14804714 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -6,7 +6,6 @@ #include #include #include -#include #include namespace mooncake { @@ -67,6 +66,7 @@ class OpLogManager { // Current number of entries in the buffer. size_t GetEntryCount() const; + private: static uint64_t NowMs(); static uint32_t ComputeChecksum(const std::string& data); @@ -77,8 +77,9 @@ class OpLogManager { uint64_t first_seq_id_{1}; // sequence_id of buffer_.front() uint64_t last_seq_id_{0}; // last assigned sequence_id - // Track per-key sequence ID for ordering guarantee - std::unordered_map key_sequence_map_; + // Note: We removed key_sequence_map_ and key_remove_time_map_. + // Global sequence_id is sufficient for ordering guarantee. + // All operations are applied in sequence_id order, which ensures consistency. // Optional etcd OpLog store for persistent storage std::shared_ptr etcd_oplog_store_; diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index b3c9bf49d1..b4e699341e 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -37,51 +37,8 @@ EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const { } bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { - // Check sequence order - bool order_valid = CheckSequenceOrder(entry); - - // If key-level sequence order violation detected, delete the key - if (!order_valid) { - // Check if it's a key-level violation (not just global sequence violation) - bool is_key_violation = false; - uint64_t current_key_seq_id = 0; - { - std::lock_guard lock(key_sequence_mutex_); - auto it = key_sequence_map_.find(entry.object_key); - if (it != key_sequence_map_.end()) { - current_key_seq_id = it->second; - // Key exists and key_sequence_id is not greater - // Also check that global sequence is correct (only handle key-level violations) - if (entry.sequence_id == expected_sequence_id_ && - entry.key_sequence_id <= current_key_seq_id) { - is_key_violation = true; - } - } - } - - if (is_key_violation) { - // Key-level sequence violation: delete the key and its metadata - LOG(WARNING) << "OpLogApplier: key sequence order violation detected, " - << "deleting key=" << entry.object_key - << ", new key_sequence_id=" << entry.key_sequence_id - << ", current key_sequence_id=" << current_key_seq_id; - - // Delete from metadata store - metadata_store_->Remove(entry.object_key); - - // Delete from key_sequence_map_ - { - std::lock_guard lock(key_sequence_mutex_); - key_sequence_map_.erase(entry.object_key); - } - - // Now the key is deleted, we can continue to process this entry - // since global sequence is correct (we checked above) - order_valid = true; - } - } - - if (!order_valid) { + // Check global sequence order (key_sequence_id is no longer used) + if (entry.sequence_id != expected_sequence_id_) { // Global sequence violation - add to pending entries std::lock_guard lock(pending_mutex_); @@ -94,10 +51,10 @@ bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { } pending_entries_[entry.sequence_id] = entry; - LOG(WARNING) << "OpLogApplier: global sequence order violation, sequence_id=" - << entry.sequence_id << ", expected=" << expected_sequence_id_ - << ", key=" << entry.object_key - << ", added to pending entries (total: " << pending_entries_.size() << ")"; + VLOG(1) << "OpLogApplier: sequence order violation, sequence_id=" + << entry.sequence_id << ", expected=" << expected_sequence_id_ + << ", key=" << entry.object_key + << ", added to pending entries (total: " << pending_entries_.size() << ")"; return false; } @@ -123,12 +80,6 @@ bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { // Update expected sequence ID expected_sequence_id_ = entry.sequence_id + 1; - // Update key sequence ID - { - std::lock_guard lock(key_sequence_mutex_); - key_sequence_map_[entry.object_key] = entry.key_sequence_id; - } - // Try to process pending entries ProcessPendingEntries(); @@ -146,11 +97,9 @@ size_t OpLogApplier::ApplyOpLogEntries(const std::vector& entries) { } uint64_t OpLogApplier::GetKeySequenceId(const std::string& key) const { - std::lock_guard lock(key_sequence_mutex_); - auto it = key_sequence_map_.find(key); - if (it != key_sequence_map_.end()) { - return it->second; - } + // Deprecated: key_sequence_id is no longer tracked. + // Global sequence_id is used for ordering. + (void)key; // Suppress unused parameter warning return 0; } @@ -239,12 +188,6 @@ size_t OpLogApplier::ProcessPendingEntries() { // Update expected sequence ID expected_sequence_id_ = entry_copy.sequence_id + 1; - // Update key sequence ID - { - std::lock_guard key_lock(key_sequence_mutex_); - key_sequence_map_[entry_copy.object_key] = entry_copy.key_sequence_id; - } - // Remove from missing list if it was there missing_sequence_ids_.erase(entry_copy.sequence_id); @@ -279,35 +222,9 @@ size_t OpLogApplier::ProcessPendingEntries() { } bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { - // Check global sequence order - if (entry.sequence_id != expected_sequence_id_) { - return false; - } - - // Check per-key sequence order - { - std::lock_guard lock(key_sequence_mutex_); - auto it = key_sequence_map_.find(entry.object_key); - if (it != key_sequence_map_.end()) { - // Key exists - check that new key_sequence_id is greater - if (entry.key_sequence_id <= it->second) { - LOG(WARNING) << "OpLogApplier: key sequence order violation, key=" - << entry.object_key - << ", new key_sequence_id=" << entry.key_sequence_id - << ", current key_sequence_id=" << it->second; - return false; - } - } - // If key doesn't exist, any key_sequence_id is valid (should be >= 1) - if (entry.key_sequence_id < 1) { - LOG(WARNING) << "OpLogApplier: invalid key_sequence_id=" - << entry.key_sequence_id << " for new key=" - << entry.object_key; - return false; - } - } - - return true; + // Only check global sequence order. + // key_sequence_id is no longer used for ordering. + return entry.sequence_id == expected_sequence_id_; } void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) { diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index 255eebc563..81a234216c 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -31,19 +31,22 @@ uint64_t OpLogManager::Append(OpType type, const std::string& key, std::unique_lock lock(mutex_); entry.sequence_id = ++last_seq_id_; - // Track per-key sequence ID for ordering guarantee - entry.key_sequence_id = ++key_sequence_map_[key]; + // Note: We use global sequence_id for ordering guarantee. + // key_sequence_id is set to sequence_id for backward compatibility, + // but the actual ordering is based on global sequence_id. + entry.key_sequence_id = entry.sequence_id; if (buffer_.size() >= kMaxBufferEntries_) { buffer_.pop_front(); ++first_seq_id_; } - buffer_.emplace_back(std::move(entry)); + buffer_.emplace_back(entry); // Copy entry to buffer // Write to etcd if EtcdOpLogStore is set if (etcd_oplog_store_) { // Release lock before writing to etcd to avoid blocking + // We use the original entry (before it was copied to buffer) lock.unlock(); ErrorCode err = etcd_oplog_store_->WriteOpLog(entry); if (err != ErrorCode::OK) { From af9f082840cf5aab8c0ea161645aba7dd5ed108e Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 16:42:25 +0800 Subject: [PATCH 28/59] =?UTF-8?q?=E5=BA=8F=E5=88=97=E5=8C=96=E5=92=8C?= =?UTF-8?q?=E5=8F=8D=E5=BA=8F=E5=88=97=E5=8C=96metadata:standby=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E6=81=A2=E5=A4=8D=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/include/hot_standby_service.h | 5 +- mooncake-store/include/metadata_store.h | 73 +++++++++++++++++++- mooncake-store/include/oplog_manager.h | 2 + mooncake-store/src/hot_standby_service.cpp | 24 ++++++- mooncake-store/src/master_service.cpp | 43 +++++++++++- mooncake-store/src/oplog_applier.cpp | 52 ++++++++++++-- 6 files changed, 187 insertions(+), 12 deletions(-) diff --git a/mooncake-store/include/hot_standby_service.h b/mooncake-store/include/hot_standby_service.h index 439947cd71..30cc17363a 100644 --- a/mooncake-store/include/hot_standby_service.h +++ b/mooncake-store/include/hot_standby_service.h @@ -146,15 +146,18 @@ class HotStandbyService { // Simple in-memory metadata store implementation class StandbyMetadataStore : public MetadataStore { public: + bool PutMetadata(const std::string& key, + const StandbyObjectMetadata& metadata) override; bool Put(const std::string& key, const std::string& payload = std::string()) override; + const StandbyObjectMetadata* GetMetadata(const std::string& key) const override; bool Remove(const std::string& key) override; bool Exists(const std::string& key) const override; size_t GetKeyCount() const override; private: mutable std::mutex mutex_; - std::unordered_map store_; + std::unordered_map store_; }; std::unique_ptr metadata_store_; diff --git a/mooncake-store/include/metadata_store.h b/mooncake-store/include/metadata_store.h index 3451bd248e..308fe1a636 100644 --- a/mooncake-store/include/metadata_store.h +++ b/mooncake-store/include/metadata_store.h @@ -1,10 +1,66 @@ #pragma once #include +#include #include +#include + +#include "replica.h" +#include "types.h" +#include "ylt/struct_json/json_reader.h" +#include "ylt/struct_json/json_writer.h" namespace mooncake { +/** + * @brief Metadata structure for Standby to store and restore object information + * + * This structure contains all essential metadata information needed by Standby + * to immediately serve as Primary when promoted. + */ +struct StandbyObjectMetadata { + UUID client_id{0, 0}; + uint64_t size{0}; + std::vector replicas; + uint64_t lease_timeout_ms{0}; // Lease timeout as milliseconds since epoch + std::optional soft_pin_timeout_ms; // Soft pin timeout as milliseconds since epoch + uint64_t last_sequence_id{0}; // Last OpLog sequence ID that modified this key + + StandbyObjectMetadata() = default; + + // Check if this metadata has valid replicas + bool HasReplicas() const { return !replicas.empty(); } +}; + +/** + * @brief Payload structure for JSON serialization/deserialization + * + * Uses separate fields for UUID since std::pair cannot be directly serialized. + */ +struct MetadataPayload { + uint64_t client_id_first{0}; // UUID.first + uint64_t client_id_second{0}; // UUID.second + uint64_t size{0}; + std::vector replicas; + uint64_t lease_timeout_ms{0}; + std::optional soft_pin_timeout_ms; + + YLT_REFL(MetadataPayload, client_id_first, client_id_second, size, replicas, + lease_timeout_ms, soft_pin_timeout_ms); + + // Convert to StandbyObjectMetadata + StandbyObjectMetadata ToStandbyMetadata(uint64_t sequence_id) const { + StandbyObjectMetadata meta; + meta.client_id = {client_id_first, client_id_second}; + meta.size = size; + meta.replicas = replicas; + meta.lease_timeout_ms = lease_timeout_ms; + meta.soft_pin_timeout_ms = soft_pin_timeout_ms; + meta.last_sequence_id = sequence_id; + return meta; + } +}; + /** * @brief Abstract interface for metadata storage on Standby * @@ -16,13 +72,28 @@ class MetadataStore { virtual ~MetadataStore() = default; /** - * @brief Put or update metadata for a key + * @brief Put or update metadata for a key with structured metadata + * @param key Object key + * @param metadata Structured metadata object + * @return true on success, false on failure + */ + virtual bool PutMetadata(const std::string& key, const StandbyObjectMetadata& metadata) = 0; + + /** + * @brief Put or update metadata for a key (legacy interface for backward compatibility) * @param key Object key * @param payload Optional payload data (JSON serialized metadata) * @return true on success, false on failure */ virtual bool Put(const std::string& key, const std::string& payload = std::string()) = 0; + /** + * @brief Get metadata for a key + * @param key Object key + * @return Pointer to metadata if found, nullptr otherwise + */ + virtual const StandbyObjectMetadata* GetMetadata(const std::string& key) const = 0; + /** * @brief Remove metadata for a key * @param key Object key diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index ab14804714..5c365d51ac 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -23,6 +23,8 @@ enum class OpType : uint8_t { }; // A single operation log entry. +// Note: Payload contains JSON serialized MetadataPayload (defined in metadata_store.h) +// for PUT_END operations, allowing Standby to restore complete metadata. struct OpLogEntry { uint64_t sequence_id{0}; // Monotonically increasing global sequence uint64_t timestamp_ms{0}; // Logical timestamp in milliseconds diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index a7c0cab85d..fbfe2c56ba 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -23,13 +23,35 @@ HotStandbyService::HotStandbyService(const HotStandbyConfig& config) } // StandbyMetadataStore implementation +bool HotStandbyService::StandbyMetadataStore::PutMetadata( + const std::string& key, const StandbyObjectMetadata& metadata) { + std::lock_guard lock(mutex_); + store_[key] = metadata; + VLOG(2) << "StandbyMetadataStore: stored metadata for key=" << key + << ", replicas=" << metadata.replicas.size() + << ", size=" << metadata.size; + return true; +} + bool HotStandbyService::StandbyMetadataStore::Put(const std::string& key, const std::string& payload) { + // Legacy interface - create empty metadata + StandbyObjectMetadata metadata; std::lock_guard lock(mutex_); - store_[key] = payload; // payload may be empty for now + store_[key] = metadata; return true; } +const StandbyObjectMetadata* HotStandbyService::StandbyMetadataStore::GetMetadata( + const std::string& key) const { + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it != store_.end()) { + return &it->second; + } + return nullptr; +} + bool HotStandbyService::StandbyMetadataStore::Remove(const std::string& key) { std::lock_guard lock(mutex_); auto it = store_.find(key); diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 89b9d6d82a..7e5dc283a9 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -6,16 +6,52 @@ #include #include #include +#include #include "etcd_helper.h" #include "etcd_oplog_store.h" #include "master_metric_manager.h" +#include "metadata_store.h" // For MetadataPayload #include "segment.h" #include "types.h" // replication_service.h removed - using etcd-based OpLog sync instead namespace mooncake { +/** + * @brief Serialize ObjectMetadata to JSON string for OpLog payload + * + * This function extracts essential metadata information (replicas, size, lease) + * and serializes it to JSON so that Standby can restore metadata when promoted. + * Uses MetadataPayload structure from metadata_store.h for consistency with deserialization. + */ +static std::string SerializeMetadataForOpLog(const MasterService::ObjectMetadata& metadata) { + MetadataPayload payload; + payload.client_id_first = metadata.client_id.first; + payload.client_id_second = metadata.client_id.second; + payload.size = metadata.size; + + // Extract replica descriptors + payload.replicas.reserve(metadata.replicas.size()); + for (const auto& replica : metadata.replicas) { + payload.replicas.push_back(replica.get_descriptor()); + } + + // Convert time_point to milliseconds since epoch + auto lease_duration = metadata.lease_timeout.time_since_epoch(); + payload.lease_timeout_ms = std::chrono::duration_cast(lease_duration).count(); + + if (metadata.soft_pin_timeout.has_value()) { + auto soft_pin_duration = metadata.soft_pin_timeout->time_since_epoch(); + payload.soft_pin_timeout_ms = std::chrono::duration_cast(soft_pin_duration).count(); + } + + // Serialize to JSON + std::string json_str; + struct_json::to_json(payload, json_str); + return json_str; +} + MasterService::MasterService() : MasterService(MasterServiceConfig()) {} MasterService::MasterService(const MasterServiceConfig& config) @@ -745,9 +781,10 @@ auto MasterService::PutEnd(const UUID& client_id, const std::string& key, metadata.GrantLease(0, default_kv_soft_pin_ttl_); // Record OpLog entry for PUT_END so that standbys can replay this change. - // For now we do not include extra payload; it can be extended later if - // needed (e.g. to carry replica descriptors). - AppendOpLogAndNotify(OpType::PUT_END, key); + // Serialize metadata (replicas, size, lease) to payload so Standby can restore + // complete metadata when promoted to Primary. + std::string metadata_payload = SerializeMetadataForOpLog(metadata); + AppendOpLogAndNotify(OpType::PUT_END, key, metadata_payload); return {}; } diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index b4e699341e..5a8e10cad6 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -1,6 +1,7 @@ #include "oplog_applier.h" #include +#include #include #include @@ -228,15 +229,54 @@ bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { } void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) { - // For now, payload is empty in current implementation. - // In the future, payload may contain serialized metadata. - // For now, we just mark the key as existing. - if (!metadata_store_->Put(entry.object_key, entry.payload)) { - LOG(ERROR) << "OpLogApplier: failed to Put key=" << entry.object_key + // Payload contains serialized metadata (replicas, size, lease) in JSON format. + // Deserialize the payload immediately and store structured metadata. + // This allows Standby to serve requests immediately after promotion. + + if (entry.payload.empty()) { + // No payload - create empty metadata (legacy compatibility) + LOG(WARNING) << "OpLogApplier: PUT_END without payload, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + StandbyObjectMetadata empty_metadata; + empty_metadata.last_sequence_id = entry.sequence_id; + if (!metadata_store_->PutMetadata(entry.object_key, empty_metadata)) { + LOG(ERROR) << "OpLogApplier: failed to PutMetadata key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } + return; + } + + // Deserialize payload to MetadataPayload + MetadataPayload payload; + bool parse_success = false; + try { + struct_json::from_json(payload, entry.payload); + parse_success = true; + } catch (const std::exception& e) { + LOG(ERROR) << "OpLogApplier: failed to parse payload for key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << ", error=" << e.what(); + } + + if (!parse_success) { + // Fallback to empty metadata if parsing fails + StandbyObjectMetadata empty_metadata; + empty_metadata.last_sequence_id = entry.sequence_id; + metadata_store_->PutMetadata(entry.object_key, empty_metadata); + return; + } + + // Convert to StandbyObjectMetadata and store + StandbyObjectMetadata metadata = payload.ToStandbyMetadata(entry.sequence_id); + + if (!metadata_store_->PutMetadata(entry.object_key, metadata)) { + LOG(ERROR) << "OpLogApplier: failed to PutMetadata key=" << entry.object_key << ", sequence_id=" << entry.sequence_id; } else { VLOG(1) << "OpLogApplier: applied PUT_END, key=" << entry.object_key - << ", sequence_id=" << entry.sequence_id; + << ", sequence_id=" << entry.sequence_id + << ", replicas=" << metadata.replicas.size() + << ", size=" << metadata.size; } } From 16e3bf1db745d88deb2c799b02c58031bad068c7 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 16:44:19 +0800 Subject: [PATCH 29/59] fix --- mooncake-store/include/master_service.h | 7 +++++++ mooncake-store/src/master_service.cpp | 13 +++---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 78da79b334..2d9893f646 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -320,6 +320,13 @@ class MasterService { void AppendOpLogAndNotify(OpType type, const std::string& key, const std::string& payload = std::string()); + /** + * @brief Serialize ObjectMetadata to JSON string for OpLog payload + * @param metadata The metadata to serialize + * @return JSON string containing the serialized metadata + */ + std::string SerializeMetadataForOpLog(const ObjectMetadata& metadata) const; + // Resolve the key to a sanitized format for storage std::string SanitizeKey(const std::string& key) const; std::string ResolvePath(const std::string& key) const; diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 7e5dc283a9..5705b41c0d 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -18,14 +18,9 @@ namespace mooncake { -/** - * @brief Serialize ObjectMetadata to JSON string for OpLog payload - * - * This function extracts essential metadata information (replicas, size, lease) - * and serializes it to JSON so that Standby can restore metadata when promoted. - * Uses MetadataPayload structure from metadata_store.h for consistency with deserialization. - */ -static std::string SerializeMetadataForOpLog(const MasterService::ObjectMetadata& metadata) { +MasterService::MasterService() : MasterService(MasterServiceConfig()) {} + +std::string MasterService::SerializeMetadataForOpLog(const ObjectMetadata& metadata) const { MetadataPayload payload; payload.client_id_first = metadata.client_id.first; payload.client_id_second = metadata.client_id.second; @@ -52,8 +47,6 @@ static std::string SerializeMetadataForOpLog(const MasterService::ObjectMetadata return json_str; } -MasterService::MasterService() : MasterService(MasterServiceConfig()) {} - MasterService::MasterService(const MasterServiceConfig& config) : default_kv_lease_ttl_(config.default_kv_lease_ttl), default_kv_soft_pin_ttl_(config.default_kv_soft_pin_ttl), From d24cfcb7a5b570eac1a86a57d8597500c8b83137 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 16:50:10 +0800 Subject: [PATCH 30/59] fix --- mooncake-store/include/master_service.h | 14 +++++++------- mooncake-store/include/replica.h | 22 ++++++++++++++++------ mooncake-store/src/master_service.cpp | 2 +- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 2d9893f646..b8b65cd6ba 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -320,13 +320,6 @@ class MasterService { void AppendOpLogAndNotify(OpType type, const std::string& key, const std::string& payload = std::string()); - /** - * @brief Serialize ObjectMetadata to JSON string for OpLog payload - * @param metadata The metadata to serialize - * @return JSON string containing the serialized metadata - */ - std::string SerializeMetadataForOpLog(const ObjectMetadata& metadata) const; - // Resolve the key to a sanitized format for storage std::string SanitizeKey(const std::string& key) const; std::string ResolvePath(const std::string& key) const; @@ -503,6 +496,13 @@ class MasterService { } }; + /** + * @brief Serialize ObjectMetadata to JSON string for OpLog payload + * @param metadata The metadata to serialize + * @return JSON string containing the serialized metadata + */ + std::string SerializeMetadataForOpLog(const ObjectMetadata& metadata) const; + static constexpr size_t kNumShards = 1024; // Number of metadata shards // Sharded metadata maps and their mutexes diff --git a/mooncake-store/include/replica.h b/mooncake-store/include/replica.h index 5793e90f96..997259ca0b 100644 --- a/mooncake-store/include/replica.h +++ b/mooncake-store/include/replica.h @@ -131,10 +131,21 @@ struct DiskDescriptor { }; struct LocalDiskDescriptor { - UUID client_id; + uint64_t client_id_first{0}; // UUID.first - split for JSON serialization + uint64_t client_id_second{0}; // UUID.second - split for JSON serialization uint64_t object_size = 0; std::string transport_endpoint; - YLT_REFL(LocalDiskDescriptor, client_id, object_size, transport_endpoint); + + // Constructor from UUID for convenience + LocalDiskDescriptor() = default; + LocalDiskDescriptor(UUID client_id, uint64_t object_size, const std::string& transport_endpoint) + : client_id_first(client_id.first), client_id_second(client_id.second), + object_size(object_size), transport_endpoint(transport_endpoint) {} + + // Get UUID (for backward compatibility) + UUID GetClientId() const { return {client_id_first, client_id_second}; } + + YLT_REFL(LocalDiskDescriptor, client_id_first, client_id_second, object_size, transport_endpoint); }; class Replica { @@ -375,10 +386,9 @@ inline Replica::Descriptor Replica::get_descriptor() const { desc.descriptor_variant = std::move(disk_desc); } else if (is_local_disk_replica()) { const auto& disk_data = std::get(data_); - LocalDiskDescriptor local_disk_desc; - local_disk_desc.client_id = disk_data.client_id; - local_disk_desc.object_size = disk_data.object_size; - local_disk_desc.transport_endpoint = disk_data.transport_endpoint; + LocalDiskDescriptor local_disk_desc(disk_data.client_id, + disk_data.object_size, + disk_data.transport_endpoint); desc.descriptor_variant = std::move(local_disk_desc); } diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 5705b41c0d..d0ad2541e4 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -802,7 +802,7 @@ auto MasterService::AddReplica(const UUID& client_id, const std::string& key, auto& descriptor = metadata.replicas[i] .get_descriptor() .get_local_disk_descriptor(); - if (descriptor.client_id == client_id) { + if (descriptor.GetClientId() == client_id) { update = true; descriptor.transport_endpoint = replica.get_descriptor() .get_local_disk_descriptor() From 19b6f0781dbda9b9afd0b6d7a965c34aa7d31a98 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 19:32:17 +0800 Subject: [PATCH 31/59] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=8A=E6=8A=A5seqid?= =?UTF-8?q?=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/include/etcd_oplog_store.h | 37 ++++++ mooncake-store/include/hot_standby_service.h | 10 ++ mooncake-store/include/oplog_manager.h | 4 + mooncake-store/src/etcd_oplog_store.cpp | 84 ++++++++++++-- mooncake-store/src/hot_standby_service.cpp | 115 ++++++++++++++++--- mooncake-store/src/oplog_manager.cpp | 13 +++ 6 files changed, 242 insertions(+), 21 deletions(-) diff --git a/mooncake-store/include/etcd_oplog_store.h b/mooncake-store/include/etcd_oplog_store.h index 5617736afd..6104976eec 100644 --- a/mooncake-store/include/etcd_oplog_store.h +++ b/mooncake-store/include/etcd_oplog_store.h @@ -1,7 +1,11 @@ #pragma once +#include +#include #include +#include #include +#include #include #include "oplog_manager.h" @@ -92,6 +96,11 @@ class EtcdOpLogStore { */ ErrorCode CleanupOpLogBefore(uint64_t before_sequence_id); + /** + * @brief Destructor - stops batch update thread. + */ + ~EtcdOpLogStore(); + private: /** * @brief Build the etcd key for an OpLog entry. @@ -129,11 +138,39 @@ class EtcdOpLogStore { bool DeserializeOpLogEntry(const std::string& json_str, OpLogEntry& entry) const; + /** + * @brief Batch update thread function. + * Periodically updates latest_sequence_id in etcd. + */ + void BatchUpdateThread(); + + /** + * @brief Trigger immediate batch update if threshold is reached. + */ + void TriggerBatchUpdateIfNeeded(); + + /** + * @brief Perform the actual batch update to etcd. + */ + void DoBatchUpdate(); + std::string cluster_id_; static constexpr const char* kOpLogPrefix = "/oplog/"; static constexpr const char* kLatestSuffix = "/latest"; static constexpr const char* kSnapshotPrefix = "/oplog/"; static constexpr const char* kSnapshotSuffix = "/snapshot/"; + + // Batch update mechanism for latest_sequence_id + std::atomic pending_latest_seq_id_{0}; + std::atomic pending_count_{0}; + std::atomic batch_update_running_{false}; + std::mutex batch_update_mutex_; + std::thread batch_update_thread_; + std::chrono::steady_clock::time_point last_update_time_; + + // Batch update configuration + static constexpr size_t kBatchSize = 100; // Update every 100 entries + static constexpr int kBatchIntervalMs = 1000; // Or every 1 second }; } // namespace mooncake diff --git a/mooncake-store/include/hot_standby_service.h b/mooncake-store/include/hot_standby_service.h index 30cc17363a..3de61928be 100644 --- a/mooncake-store/include/hot_standby_service.h +++ b/mooncake-store/include/hot_standby_service.h @@ -106,6 +106,16 @@ class HotStandbyService { */ size_t GetMetadataCount() const; + /** + * @brief Get the latest applied sequence ID after promotion + * + * This should be called after Promote() to get the sequence_id + * that the new Primary's OpLogManager should start from. + * + * @return Latest applied sequence ID, or 0 if not available + */ + uint64_t GetLatestAppliedSequenceId() const; + private: /** * @brief Main replication loop (runs in background thread) diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index 5c365d51ac..4a343360be 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -62,6 +62,10 @@ class OpLogManager { // Get the latest assigned sequence id. Returns 0 if no entry exists. uint64_t GetLastSequenceId() const; + // Set the initial sequence ID (used when promoting Standby to Primary). + // This ensures the new Primary's OpLogManager continues from the correct sequence_id. + void SetInitialSequenceId(uint64_t sequence_id); + // Truncate all entries with sequence_id < min_seq_to_keep. void TruncateBefore(uint64_t min_seq_to_keep); diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp index 8a86cddf75..de200715e4 100644 --- a/mooncake-store/src/etcd_oplog_store.cpp +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -14,7 +14,25 @@ namespace mooncake { EtcdOpLogStore::EtcdOpLogStore(const std::string& cluster_id) - : cluster_id_(cluster_id) {} + : cluster_id_(cluster_id), + last_update_time_(std::chrono::steady_clock::now()) { + // Start batch update thread + batch_update_running_.store(true); + batch_update_thread_ = std::thread(&EtcdOpLogStore::BatchUpdateThread, this); +} + +EtcdOpLogStore::~EtcdOpLogStore() { + // Stop batch update thread + batch_update_running_.store(false); + if (batch_update_thread_.joinable()) { + batch_update_thread_.join(); + } + + // Perform final update if there are pending updates + if (pending_count_.load() > 0) { + DoBatchUpdate(); + } +} ErrorCode EtcdOpLogStore::WriteOpLog(const OpLogEntry& entry) { std::string key = BuildOpLogKey(entry.sequence_id); @@ -28,12 +46,13 @@ ErrorCode EtcdOpLogStore::WriteOpLog(const OpLogEntry& entry) { return err; } - // Update latest sequence_id - err = UpdateLatestSequenceId(entry.sequence_id); - if (err != ErrorCode::OK) { - LOG(WARNING) << "Failed to update latest sequence_id, but OpLog entry " - "was written successfully"; - // Don't return error here, as the OpLog entry was written + // Add to batch update queue instead of immediate update + pending_latest_seq_id_.store(entry.sequence_id); + size_t count = pending_count_.fetch_add(1) + 1; + + // Trigger immediate update if batch size threshold is reached + if (count >= kBatchSize) { + DoBatchUpdate(); } return ErrorCode::OK; @@ -217,5 +236,56 @@ bool EtcdOpLogStore::DeserializeOpLogEntry(const std::string& json_str, return true; } +void EtcdOpLogStore::BatchUpdateThread() { + while (batch_update_running_.load()) { + std::this_thread::sleep_for( + std::chrono::milliseconds(kBatchIntervalMs)); + + // Check if we need to update based on time interval + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast( + now - last_update_time_).count(); + + if (pending_count_.load() > 0 && elapsed >= kBatchIntervalMs) { + DoBatchUpdate(); + } + } +} + +void EtcdOpLogStore::TriggerBatchUpdateIfNeeded() { + // This method is kept for potential future use (e.g., manual trigger) + // Currently, DoBatchUpdate() is called directly from WriteOpLog + // when batch size threshold is reached + if (pending_count_.load() >= kBatchSize) { + DoBatchUpdate(); + } +} + +void EtcdOpLogStore::DoBatchUpdate() { + std::lock_guard lock(batch_update_mutex_); + + // Get the pending sequence_id and reset counters + uint64_t seq_id_to_update = pending_latest_seq_id_.load(); + size_t count = pending_count_.exchange(0); + + if (count == 0) { + return; // Nothing to update + } + + // Update latest_sequence_id in etcd + ErrorCode err = UpdateLatestSequenceId(seq_id_to_update); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to batch update latest_sequence_id=" + << seq_id_to_update << ", error=" << err + << ". Will retry in next batch."; + // Restore the count so it will be retried + pending_count_.fetch_add(count); + } else { + last_update_time_ = std::chrono::steady_clock::now(); + VLOG(2) << "Batch updated latest_sequence_id=" << seq_id_to_update + << " (count=" << count << " entries)"; + } +} + } // namespace mooncake diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index fbfe2c56ba..c0905783ae 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -212,36 +212,112 @@ bool HotStandbyService::IsReadyForPromotion() const { return false; } - // Check if lag is within threshold - return status.lag_entries <= config_.max_replication_lag_entries; + // Allow promotion even with large lag - the new Primary can continue + // syncing remaining OpLog entries from etcd after promotion. + // Log a warning if lag is large, but don't block promotion. + if (status.lag_entries > config_.max_replication_lag_entries) { + LOG(WARNING) << "Standby has large replication lag: " << status.lag_entries + << " entries (threshold: " << config_.max_replication_lag_entries + << "). Promotion will proceed, but remaining OpLog entries " + << "will be synced after promotion."; + } + + return true; } std::unique_ptr HotStandbyService::Promote() { std::lock_guard lock(mutex_); if (!IsReadyForPromotion()) { - LOG(ERROR) << "Standby is not ready for promotion. Lag: " - << GetSyncStatus().lag_entries << " entries"; + LOG(ERROR) << "Standby is not ready for promotion (not connected)"; return nullptr; } + StandbySyncStatus status = GetSyncStatus(); + uint64_t current_applied_seq_id = status.applied_seq_id; + LOG(INFO) << "Promoting Standby to Primary. Applied seq_id: " - << applied_seq_id_.load(); + << current_applied_seq_id + << ", lag: " << status.lag_entries << " entries"; + + // Continue syncing remaining OpLog entries from etcd before promotion + if (status.lag_entries > 0) { + LOG(INFO) << "Syncing remaining " << status.lag_entries + << " OpLog entries from etcd before promotion..."; + + // Get latest sequence_id from etcd + EtcdOpLogStore oplog_store(cluster_id_); + uint64_t latest_seq_id = 0; + ErrorCode err = oplog_store.GetLatestSequenceId(latest_seq_id); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to get latest sequence_id from etcd: " << err + << ". Will proceed with promotion, but metadata may be incomplete."; + } else { + // Read and apply remaining OpLog entries + uint64_t remaining_count = latest_seq_id - current_applied_seq_id; + if (remaining_count > 0) { + LOG(INFO) << "Reading " << remaining_count + << " remaining OpLog entries from etcd..."; + + std::vector remaining_entries; + // Read in batches to avoid memory issues + const size_t batch_size = 1000; + uint64_t start_seq = current_applied_seq_id + 1; + size_t total_applied = 0; + + while (start_seq <= latest_seq_id) { + std::vector batch; + ErrorCode read_err = oplog_store.ReadOpLogSince( + start_seq - 1, batch_size, batch); + + if (read_err != ErrorCode::OK) { + LOG(ERROR) << "Failed to read OpLog batch starting from " + << start_seq << ": " << read_err; + break; + } + + if (batch.empty()) { + break; // No more entries + } + + // Apply batch + size_t applied = oplog_applier_->ApplyOpLogEntries(batch); + total_applied += applied; + + LOG(INFO) << "Applied " << applied << " OpLog entries " + << "(batch: " << batch[0].sequence_id + << " to " << batch.back().sequence_id << ")"; + + start_seq = batch.back().sequence_id + 1; + } + + LOG(INFO) << "Completed syncing remaining OpLog entries. " + << "Total applied: " << total_applied; + } + } + } - // Stop replication + // Stop replication (OpLogWatcher will stop watching) Stop(); // In full implementation, we would: - // 1. Create a new MasterService instance + // 1. Create a new MasterService instance with appropriate config // 2. Initialize it with the replicated metadata from metadata_store_ - // 3. Return the MasterService instance - - // For now, this is a placeholder - // TODO: Implement full promotion logic - auto master_service = std::make_unique(); + // 3. Set the OpLogManager's initial sequence_id to latest_seq_id + // 4. Return the MasterService instance - LOG(INFO) << "Standby promoted to Primary successfully"; - return master_service; + // For now, this is a placeholder - the actual MasterService creation + // happens in MasterServiceSupervisor::Start() after leader election. + // This method ensures all remaining OpLog entries are synced before + // the new Primary starts serving requests. + + LOG(INFO) << "Standby promoted to Primary successfully. " + << "All remaining OpLog entries have been synced."; + + // Return nullptr - actual MasterService creation happens externally + // The caller (MasterServiceSupervisor) will create the MasterService + // with the appropriate configuration. + return nullptr; } size_t HotStandbyService::GetMetadataCount() const { @@ -249,6 +325,17 @@ size_t HotStandbyService::GetMetadataCount() const { return metadata_store_ ? metadata_store_->GetKeyCount() : 0; } +uint64_t HotStandbyService::GetLatestAppliedSequenceId() const { + std::lock_guard lock(mutex_); + if (oplog_applier_) { + uint64_t expected_seq = oplog_applier_->GetExpectedSequenceId(); + // GetExpectedSequenceId returns the next expected sequence_id, + // so the latest applied is expected_seq - 1 + return expected_seq > 0 ? expected_seq - 1 : 0; + } + return applied_seq_id_.load(); +} + void HotStandbyService::ReplicationLoop() { LOG(INFO) << "Replication loop started (etcd-based OpLog sync)"; diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index 81a234216c..7903f79c46 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -86,6 +86,19 @@ uint64_t OpLogManager::GetLastSequenceId() const { return last_seq_id_; } +void OpLogManager::SetInitialSequenceId(uint64_t sequence_id) { + std::unique_lock lock(mutex_); + if (last_seq_id_ == 0 && buffer_.empty()) { + // Only allow setting initial sequence_id if OpLogManager is empty + last_seq_id_ = sequence_id; + first_seq_id_ = sequence_id + 1; // first_seq_id_ should be > last_seq_id_ when empty + LOG(INFO) << "OpLogManager initial sequence_id set to " << sequence_id; + } else { + LOG(WARNING) << "Cannot set initial sequence_id: OpLogManager is not empty " + << "(last_seq_id_=" << last_seq_id_ << ", buffer_size=" << buffer_.size() << ")"; + } +} + void OpLogManager::TruncateBefore(uint64_t min_seq_to_keep) { std::unique_lock lock(mutex_); while (!buffer_.empty() && buffer_.front().sequence_id < min_seq_to_keep) { From 972b20a4385e37d5a6e4f7a0cdfad79cf4be1578 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 19:46:33 +0800 Subject: [PATCH 32/59] delete --- mooncake-store/include/oplog_manager.h | 6 +----- mooncake-store/src/oplog_manager.cpp | 27 +++----------------------- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index 4a343360be..31a924f29b 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -32,7 +32,7 @@ struct OpLogEntry { std::string object_key; // Target object key std::string payload; // Serialized extra data (optional) uint32_t checksum{0}; // Checksum of payload (implementation-defined) - uint32_t prefix_hash{0}; // Hash of key prefix (for future verification) + uint32_t prefix_hash{0}; // Hash of the entire key (for verification and optimization) uint64_t key_sequence_id{0}; // Per-key sequence ID (for ordering guarantee) }; @@ -55,10 +55,6 @@ class OpLogManager { uint64_t Append(OpType type, const std::string& key, const std::string& payload = std::string()); - // Get entries with sequence_id > since_seq_id, up to at most limit entries. - std::vector GetEntriesSince(uint64_t since_seq_id, - size_t limit = 1000) const; - // Get the latest assigned sequence id. Returns 0 if no entry exists. uint64_t GetLastSequenceId() const; diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index 7903f79c46..94fa6d6240 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -61,26 +61,6 @@ uint64_t OpLogManager::Append(OpType type, const std::string& key, return last_seq_id_; } -std::vector OpLogManager::GetEntriesSince(uint64_t since_seq_id, - size_t limit) const { - std::shared_lock lock(mutex_); - std::vector result; - if (buffer_.empty() || since_seq_id >= last_seq_id_) { - return result; - } - - result.reserve(std::min(limit, buffer_.size())); - for (const auto& e : buffer_) { - if (e.sequence_id > since_seq_id) { - result.push_back(e); - if (result.size() >= limit) { - break; - } - } - } - return result; -} - uint64_t OpLogManager::GetLastSequenceId() const { std::shared_lock lock(mutex_); return last_seq_id_; @@ -128,10 +108,9 @@ uint32_t OpLogManager::ComputePrefixHash(const std::string& key) { if (key.empty()) { return 0; } - // Use at most first 8 characters to compute a simple hash. - const size_t prefix_len = std::min(8, key.size()); - return static_cast( - std::hash{}(std::string_view(key.data(), prefix_len))); + // Compute hash for the entire key to avoid hash collisions. + // Using the full key ensures better distribution and fewer collisions. + return static_cast(std::hash{}(key)); } } // namespace mooncake From 71cfb8da0150fdff053c5ec62ecb17264cfcb8f1 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 29 Dec 2025 20:01:42 +0800 Subject: [PATCH 33/59] fix hash --- mooncake-store/include/oplog_manager.h | 3 --- mooncake-store/src/oplog_manager.cpp | 16 ++++------------ 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index 31a924f29b..b0f3dcdb1c 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -62,9 +62,6 @@ class OpLogManager { // This ensures the new Primary's OpLogManager continues from the correct sequence_id. void SetInitialSequenceId(uint64_t sequence_id); - // Truncate all entries with sequence_id < min_seq_to_keep. - void TruncateBefore(uint64_t min_seq_to_keep); - // Current number of entries in the buffer. size_t GetEntryCount() const; diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index 94fa6d6240..3734b3532d 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -79,14 +78,6 @@ void OpLogManager::SetInitialSequenceId(uint64_t sequence_id) { } } -void OpLogManager::TruncateBefore(uint64_t min_seq_to_keep) { - std::unique_lock lock(mutex_); - while (!buffer_.empty() && buffer_.front().sequence_id < min_seq_to_keep) { - buffer_.pop_front(); - ++first_seq_id_; - } -} - size_t OpLogManager::GetEntryCount() const { std::shared_lock lock(mutex_); return buffer_.size(); @@ -108,9 +99,10 @@ uint32_t OpLogManager::ComputePrefixHash(const std::string& key) { if (key.empty()) { return 0; } - // Compute hash for the entire key to avoid hash collisions. - // Using the full key ensures better distribution and fewer collisions. - return static_cast(std::hash{}(key)); + // Use XXH32 for consistency with ComputeChecksum and better performance. + // XXH32 provides faster hashing and lower collision rate than std::hash. + // Computing hash for the entire key ensures better distribution and fewer collisions. + return static_cast(XXH32(key.data(), key.size(), 0)); } } // namespace mooncake From 6e5e68b4e677b663cdb4bd8b8447b9a0bedcc4ee Mon Sep 17 00:00:00 2001 From: BernardLee Date: Wed, 31 Dec 2025 15:43:57 +0800 Subject: [PATCH 34/59] refactor --- .../etcd-hot-standby-architecture.puml | 139 ++++++++++ .../etcd-hot-standby-diagrams-README.md | 127 +++++++++ doc/zh/diagrams/etcd-hot-standby-flow.puml | 173 ++++++++++++ .../diagrams/etcd-hot-standby-sequence.puml | 259 ++++++++++++++++++ mooncake-common/etcd/etcd_wrapper.go | 237 ++++++++++++++++ mooncake-store/include/etcd_helper.h | 44 +++ mooncake-store/include/etcd_oplog_store.h | 19 +- mooncake-store/include/hot_standby_service.h | 22 ++ mooncake-store/include/master_service.h | 18 ++ mooncake-store/include/metadata_store.h | 13 +- mooncake-store/include/oplog_watcher.h | 22 ++ mooncake-store/include/rpc_service.h | 7 + mooncake-store/include/snapshot_provider.h | 55 ++++ mooncake-store/src/etcd_helper.cpp | 109 ++++++++ mooncake-store/src/etcd_oplog_store.cpp | 204 ++++++++++++-- mooncake-store/src/ha_helper.cpp | 31 ++- mooncake-store/src/hot_standby_service.cpp | 200 ++++++++------ mooncake-store/src/master_service.cpp | 144 +++++++++- mooncake-store/src/oplog_applier.cpp | 99 ++++--- mooncake-store/src/oplog_watcher.cpp | 155 ++++++++++- mooncake-store/src/rpc_service.cpp | 6 + 21 files changed, 1907 insertions(+), 176 deletions(-) create mode 100644 doc/zh/diagrams/etcd-hot-standby-architecture.puml create mode 100644 doc/zh/diagrams/etcd-hot-standby-diagrams-README.md create mode 100644 doc/zh/diagrams/etcd-hot-standby-flow.puml create mode 100644 doc/zh/diagrams/etcd-hot-standby-sequence.puml create mode 100644 mooncake-store/include/snapshot_provider.h diff --git a/doc/zh/diagrams/etcd-hot-standby-architecture.puml b/doc/zh/diagrams/etcd-hot-standby-architecture.puml new file mode 100644 index 0000000000..1b27d929ed --- /dev/null +++ b/doc/zh/diagrams/etcd-hot-standby-architecture.puml @@ -0,0 +1,139 @@ +@startuml etcd-hot-standby-architecture +!theme plain +skinparam componentStyle rectangle +skinparam linetype ortho + +title etcd热备架构图 + +package "Primary Master" { + component [MasterService] as MasterService { + + AppendOpLogAndNotify() + + SerializeMetadataForOpLog() + + RestoreFromStandbySnapshot() + } + + component [OpLogManager] as OpLogManager { + + Append() + + SetEtcdOpLogStore() + + SetInitialSequenceId() + } + + component [EtcdOpLogStore] as EtcdOpLogStore { + + WriteOpLog() + + UpdateLatestSequenceId() + + CleanupOpLogBefore() + } +} + +package "Standby Master" { + component [HotStandbyService] as HotStandbyService { + + Start() + + Stop() + + Promote() + + GetSyncStatus() + + ExportMetadataSnapshot() + } + + component [OpLogWatcher] as OpLogWatcher { + + StartFromSequenceId() + + WatchOpLog() + + ReadOpLogSinceWithRevision() + } + + component [OpLogApplier] as OpLogApplier { + + ApplyOpLogEntry() + + ApplyOpLogEntries() + + ProcessPendingEntries() + + RequestMissingOpLog() + } + + component [StandbyMetadataStore] as StandbyMetadataStore { + + PutMetadata() + + Remove() + + Snapshot() + } +} + +package "协调组件" { + component [MasterServiceSupervisor] as Supervisor { + + Start() + + StartStandbyService() + } + + component [MasterViewHelper] as ViewHelper { + + ElectLeader() + + KeepLeader() + } + + component [EtcdHelper] as EtcdHelper { + + ConnectToEtcdStoreClient() + + GetRangeAsJson() + + WatchWithPrefixFromRevisionV2() + + GrantLease() + + CreateWithLease() + } +} + +cloud "etcd" { + database "OpLog Storage" as OpLogStorage { + + /oplog/{cluster_id}/{sequence_id} + + /oplog/{cluster_id}/latest + } + + database "Leader Election" as LeaderElection { + + /mooncake-store/{cluster_id}/master_view + } +} + +' Primary Master 内部关系 +MasterService --> OpLogManager : 生成OpLog +OpLogManager --> EtcdOpLogStore : 写入etcd +EtcdOpLogStore --> OpLogStorage : 存储OpLog + +' Standby Master 内部关系 +HotStandbyService --> OpLogWatcher : 启动watch +HotStandbyService --> OpLogApplier : 应用OpLog +HotStandbyService --> StandbyMetadataStore : 存储metadata +OpLogWatcher --> OpLogApplier : 转发OpLog事件 +OpLogApplier --> StandbyMetadataStore : 更新metadata + +' Standby 与 etcd 关系 +OpLogWatcher --> OpLogStorage : Watch + Read +OpLogApplier --> OpLogStorage : 请求缺失OpLog + +' 协调组件关系 +Supervisor --> ViewHelper : Leader选举 +Supervisor --> MasterService : 启动Primary +Supervisor --> HotStandbyService : 启动Standby +ViewHelper --> LeaderElection : 选举Leader +ViewHelper --> EtcdHelper : etcd操作 +EtcdOpLogStore --> EtcdHelper : etcd操作 +OpLogWatcher --> EtcdHelper : etcd操作 + +' 故障切换流程 +HotStandbyService ..> Supervisor : Promote()后返回metadata +Supervisor ..> MasterService : RestoreFromStandbySnapshot() + +note right of OpLogStorage + OpLog存储格式: + Key: /oplog/{cluster_id}/{sequence_id} + Value: JSON序列化的OpLogEntry + 包含: op_type, object_key, payload等 +end note + +note right of LeaderElection + Leader选举: + - 使用etcd lease机制 + - TTL: 5秒 + - 通过CreateWithLease竞争 +end note + +note bottom of OpLogApplier + 顺序保证: + - 使用全局sequence_id保证顺序 + - 乱序的OpLog会进入pending队列 + - 缺失的OpLog会从etcd请求 +end note + +@enduml + diff --git a/doc/zh/diagrams/etcd-hot-standby-diagrams-README.md b/doc/zh/diagrams/etcd-hot-standby-diagrams-README.md new file mode 100644 index 0000000000..b970339898 --- /dev/null +++ b/doc/zh/diagrams/etcd-hot-standby-diagrams-README.md @@ -0,0 +1,127 @@ +# etcd热备架构图表说明 + +本文档包含基于当前代码实现的etcd热备架构的PlantUML图表。 + +## 图表文件 + +### 1. `etcd-hot-standby-architecture.puml` - 整体架构图 + +展示了etcd热备系统的整体架构，包括： + +- **Primary Master组件**: + - `MasterService`: 核心服务，处理客户端请求 + - `OpLogManager`: 生成和管理OpLog + - `EtcdOpLogStore`: 将OpLog写入etcd + +- **Standby Master组件**: + - `HotStandbyService`: Standby服务主控制器 + - `OpLogWatcher`: 从etcd监听OpLog变化 + - `OpLogApplier`: 应用OpLog到本地metadata store + - `StandbyMetadataStore`: Standby的metadata存储 + +- **协调组件**: + - `MasterServiceSupervisor`: 管理Primary/Standby切换 + - `MasterViewHelper`: 处理Leader选举 + - `EtcdHelper`: etcd操作的C++ wrapper + +- **etcd存储**: + - OpLog存储: `/oplog/{cluster_id}/{sequence_id}` + - Leader选举: `/mooncake-store/{cluster_id}/master_view` + +### 2. `etcd-hot-standby-sequence.puml` - 时序图 + +展示了关键流程的时序关系，包括： + +1. **Primary启动流程**: + - Leader选举 + - MasterService初始化 + - OpLogManager设置EtcdOpLogStore + +2. **Standby启动流程**: + - 检测已有Leader + - 热启动 vs 冷启动 + - 快照加载(可选) + - 历史OpLog读取 + - Watch启动 + +3. **写入操作流程**: + - 客户端写入请求 + - Primary生成OpLog + - 写入etcd + - Standby接收并应用 + +4. **故障切换流程**: + - Leader lease过期检测 + - Standby最终同步 + - 重新选举 + - 新Primary初始化 + +### 3. `etcd-hot-standby-flow.puml` - 流程图 + +展示了数据流和控制流，包括： + +1. **OpLog写入流程**: Primary如何生成和写入OpLog +2. **Standby同步流程**: Standby如何启动和同步数据 +3. **OpLog应用流程**: Standby如何应用OpLog(包括乱序处理) +4. **故障切换流程**: Standby如何提升为Primary +5. **OpLog清理流程**: 如何清理etcd中的旧OpLog +6. **批量更新流程**: latest_sequence_id的批量更新机制 + +## 关键设计点 + +### 1. 顺序保证 +- 使用全局`sequence_id`保证OpLog顺序 +- Standby通过`expected_sequence_id`检测乱序 +- 乱序的OpLog进入`pending_entries_`队列 +- 缺失的OpLog从etcd主动请求 + +### 2. 一致性保证 +- 使用etcd revision实现"read then watch"的一致性 +- `ReadOpLogSinceWithRevision`返回revision +- Watch从`revision + 1`开始，确保不丢失事件 + +### 3. 性能优化 +- `latest_sequence_id`批量更新(每100条或每1秒) +- OpLog读取使用分页(每批1000条) +- 使用固定宽度sequence_id确保etcd key的字典序 + +### 4. 故障恢复 +- Standby提升前进行最终同步 +- 新Primary从Standby的metadata快照恢复 +- OpLog sequence_id连续，避免回退 + +## 使用方法 + +### 查看图表 + +1. **在线查看**: 使用PlantUML在线服务器 + - 访问: http://www.plantuml.com/plantuml/uml/ + - 复制`.puml`文件内容粘贴查看 + +2. **VS Code插件**: 安装PlantUML插件 + - 插件: `PlantUML` + - 打开`.puml`文件，按`Alt+D`预览 + +3. **命令行工具**: 使用PlantUML命令行工具 + ```bash + java -jar plantuml.jar etcd-hot-standby-architecture.puml + ``` + +### 导出图片 + +```bash +# 导出为PNG +java -jar plantuml.jar -tpng *.puml + +# 导出为SVG +java -jar plantuml.jar -tsvg *.puml + +# 导出为PDF +java -jar plantuml.jar -tpdf *.puml +``` + +## 相关文档 + +- [RFC: etcd热备完整方案](../rfc-oplog-hot-standby-complete.md) +- [实现计划](../rfc-oplog-implementation-plan.md) + diff --git a/doc/zh/diagrams/etcd-hot-standby-flow.puml b/doc/zh/diagrams/etcd-hot-standby-flow.puml new file mode 100644 index 0000000000..33de9cb159 --- /dev/null +++ b/doc/zh/diagrams/etcd-hot-standby-flow.puml @@ -0,0 +1,173 @@ +@startuml etcd-hot-standby-flow +!theme plain +skinparam activity { + BackgroundColor #E1F5FF + BorderColor #0066CC + FontColor #000000 +} +skinparam arrow { + Color #0066CC + Thickness 2 +} + +title etcd热备数据流和控制流程图 + +partition "OpLog写入流程" { +start +:Primary Master接收操作请求; +:MasterService处理请求; +:序列化metadata为JSON; +:OpLogManager.Append(); +note right + 生成: + - sequence_id (全局递增) + - timestamp_ms + - checksum + - prefix_hash +end note +:EtcdOpLogStore.WriteOpLog(); +:写入etcd: /oplog/{cluster_id}/{sequence_id}; +:触发批量更新latest_sequence_id; +note right + 批量更新策略: + - 每100条或每1秒 + - 减少etcd写入压力 +end note +stop +} + +partition "Standby同步流程" { +start +:Standby启动; +if (已有本地metadata?) then (是 - 热启动) + :读取本地last_seq_id; + :OpLogApplier.Recover(last_seq_id); +else (否 - 冷启动) + if (启用快照?) then (是) + :SnapshotProvider.LoadLatestSnapshot(); + :加载快照到StandbyMetadataStore; + :OpLogApplier.Recover(snapshot_seq_id); + endif +endif +:OpLogWatcher.StartFromSequenceId(); +:读取历史OpLog (ReadOpLogSinceWithRevision); +note right + 使用分页读取: + - 每批1000条 + - 返回etcd revision +end note +:应用OpLog到StandbyMetadataStore; +:设置next_watch_revision = revision + 1; +:启动Watch线程 (WatchWithPrefixFromRevisionV2); +:持续监听etcd OpLog变化; +stop +} + +partition "OpLog应用流程" { +start +:OpLogWatcher收到Watch事件; +:反序列化OpLogEntry; +:OpLogApplier.ApplyOpLogEntry(); +if (sequence_id == expected_sequence_id?) then (是) + :直接应用; + switch (op_type) + case (PUT_END) + :反序列化payload; + :StandbyMetadataStore.PutMetadata(); + case (PUT_REVOKE) + :StandbyMetadataStore.Remove(); + case (REMOVE) + :StandbyMetadataStore.Remove(); + endswitch + :expected_sequence_id++; + :处理pending队列; +else (否 - 乱序) + if (sequence_id < expected_sequence_id?) then (是 - 重复) + :忽略(已处理); + else (否 - 超前) + :加入pending队列; + :记录missing_sequence_ids; + if (等待超过5秒?) then (是) + :RequestMissingOpLog(); + :从etcd读取缺失OpLog; + :应用缺失OpLog; + endif + endif +endif +stop +} + +partition "故障切换流程" { +start +:etcd检测到Leader lease过期; +:MasterViewHelper检测到Leader删除; +:MasterServiceSupervisor触发切换; +:HotStandbyService.Promote(); +:停止OpLogWatcher; +:最终同步: 读取剩余OpLog; +note right + 循环读取直到: + - 没有更多OpLog + - 或读取失败 +end note +:应用所有剩余OpLog; +:ExportMetadataSnapshot(); +:GetLatestAppliedSequenceId(); +:MasterServiceSupervisor重新选举; +if (选举成功?) then (是) + :创建新MasterService; + :OpLogManager.SetInitialSequenceId(last_seq_id); + :MasterService.RestoreFromStandbySnapshot(); + note right + 恢复过程: + - 创建DummyBufferAllocator + - 重建Replica对象 + - 恢复metadata到本地 + - 不恢复lease信息 + end note + :启动新Primary服务; +else (否) + :继续作为Standby; +endif +stop +} + +partition "OpLog清理流程" { +start +:定期触发清理任务; +:EtcdOpLogStore.CleanupOpLogBefore(); +:查询etcd中最小sequence_id; +note right + Scheme 3: + - 不依赖持久化的"cleaned_upto" + - 查询实际最小sequence_id + - 更可靠 +end note +if (最小seq_id < before_sequence_id?) then (是) + :DeleteRange(/oplog/{cluster_id}/0, before_seq_id); + :删除etcd中的旧OpLog; +else (否) + :无需清理; +endif +stop +} + +partition "批量更新latest_sequence_id流程" { +start +:EtcdOpLogStore.WriteOpLog(); +:pending_latest_seq_id = sequence_id; +:pending_count++; +if (pending_count >= 100\n或距离上次更新 >= 1秒?) then (是) + :DoBatchUpdate(); + :UpdateLatestSequenceId(pending_latest_seq_id); + :写入etcd: /oplog/{cluster_id}/latest; + :pending_count = 0; + :last_update_time = now; +else (否) + :继续累积; +endif +stop +} + +@enduml + diff --git a/doc/zh/diagrams/etcd-hot-standby-sequence.puml b/doc/zh/diagrams/etcd-hot-standby-sequence.puml new file mode 100644 index 0000000000..ecdd53809f --- /dev/null +++ b/doc/zh/diagrams/etcd-hot-standby-sequence.puml @@ -0,0 +1,259 @@ +@startuml etcd-hot-standby-sequence +!theme plain +skinparam sequenceMessageAlign center +skinparam sequenceArrowThickness 2 + +title etcd热备关键时序图 + +== Primary启动 == + +actor User +participant Supervisor as "MasterServiceSupervisor" +participant ViewHelper as "MasterViewHelper" +participant EtcdHelper as "EtcdHelper" +database etcd as "etcd" +participant MasterService as "MasterService" +participant OpLogManager as "OpLogManager" +participant EtcdOpLogStore as "EtcdOpLogStore" + +User -> Supervisor: 启动服务 +activate Supervisor + +Supervisor -> ViewHelper: ElectLeader() +activate ViewHelper +ViewHelper -> EtcdHelper: GrantLease(TTL=5s) +EtcdHelper -> etcd: 创建lease +etcd --> EtcdHelper: lease_id +ViewHelper -> EtcdHelper: CreateWithLease(key, lease_id) +EtcdHelper -> etcd: 尝试创建leader key +alt 成功 + etcd --> EtcdHelper: 成功，成为Leader + ViewHelper --> Supervisor: 选举成功 +else 失败 + etcd --> EtcdHelper: 失败，已有Leader + ViewHelper -> EtcdHelper: WatchUntilDeleted() + EtcdHelper -> etcd: Watch leader key + etcd --> EtcdHelper: Leader删除事件 + ViewHelper --> Supervisor: Leader已删除，重试选举 +end +deactivate ViewHelper + +Supervisor -> MasterService: 创建MasterService +activate MasterService +MasterService -> OpLogManager: 创建OpLogManager +activate OpLogManager +MasterService -> EtcdOpLogStore: 创建EtcdOpLogStore(enable_batch=true) +activate EtcdOpLogStore +OpLogManager -> EtcdOpLogStore: SetEtcdOpLogStore() +deactivate EtcdOpLogStore +deactivate OpLogManager +deactivate MasterService + +Supervisor -> MasterService: 启动服务 +activate MasterService +MasterService -> ViewHelper: KeepLeader(lease_id) +activate ViewHelper +ViewHelper -> EtcdHelper: KeepAlive(lease_id) +EtcdHelper -> etcd: 定期续约 +deactivate ViewHelper +deactivate MasterService +deactivate Supervisor + +== Standby启动 == + +participant HotStandbyService as "HotStandbyService" +participant OpLogWatcher as "OpLogWatcher" +participant OpLogApplier as "OpLogApplier" +participant StandbyMetadataStore as "StandbyMetadataStore" + +User -> Supervisor: 启动服务(已有Leader) +activate Supervisor + +Supervisor -> ViewHelper: GetMasterView() +activate ViewHelper +ViewHelper -> EtcdHelper: Get(key) +EtcdHelper -> etcd: 查询leader +etcd --> EtcdHelper: 返回leader地址 +EtcdHelper --> ViewHelper: leader地址 +ViewHelper --> Supervisor: 已有Leader +deactivate ViewHelper + +Supervisor -> HotStandbyService: 创建HotStandbyService +activate HotStandbyService +HotStandbyService -> StandbyMetadataStore: 创建StandbyMetadataStore +activate StandbyMetadataStore +HotStandbyService -> OpLogApplier: 创建OpLogApplier +activate OpLogApplier +HotStandbyService -> OpLogWatcher: 创建OpLogWatcher +activate OpLogWatcher + +Supervisor -> HotStandbyService: Start(etcd_endpoints, cluster_id) +HotStandbyService -> EtcdHelper: ConnectToEtcdStoreClient() +EtcdHelper -> etcd: 连接etcd +etcd --> EtcdHelper: 连接成功 + +alt 热启动(已有metadata) + HotStandbyService -> OpLogApplier: GetExpectedSequenceId() + OpLogApplier --> HotStandbyService: last_seq_id + HotStandbyService -> OpLogApplier: Recover(last_seq_id) +else 冷启动(无metadata) + opt 启用快照 + HotStandbyService -> StandbyMetadataStore: LoadLatestSnapshot() + StandbyMetadataStore --> HotStandbyService: snapshot + snapshot_seq_id + HotStandbyService -> OpLogApplier: Recover(snapshot_seq_id) + end +end + +HotStandbyService -> OpLogWatcher: StartFromSequenceId(start_seq_id) +OpLogWatcher -> EtcdOpLogStore: ReadOpLogSinceWithRevision(start_seq_id) +activate EtcdOpLogStore +EtcdOpLogStore -> EtcdHelper: GetRangeAsJson(prefix, limit) +EtcdHelper -> etcd: Range Get +etcd --> EtcdHelper: OpLog entries + revision +EtcdHelper --> EtcdOpLogStore: entries + revision_id +EtcdOpLogStore --> OpLogWatcher: entries + revision_id +deactivate EtcdOpLogStore + +loop 批量读取历史OpLog + OpLogWatcher -> OpLogApplier: ApplyOpLogEntries(batch) + OpLogApplier -> StandbyMetadataStore: PutMetadata() / Remove() + StandbyMetadataStore --> OpLogApplier: 成功 + OpLogApplier --> OpLogWatcher: applied_count +end + +OpLogWatcher -> OpLogWatcher: next_watch_revision = revision_id + 1 +OpLogWatcher -> OpLogWatcher: WatchOpLog() [后台线程] +OpLogWatcher -> EtcdHelper: WatchWithPrefixFromRevisionV2(prefix, start_revision) +EtcdHelper -> etcd: Watch from revision +etcd --> EtcdHelper: OpLog事件流 +deactivate OpLogWatcher +deactivate OpLogApplier +deactivate StandbyMetadataStore +deactivate HotStandbyService +deactivate Supervisor + +== 写入操作流程 == + +participant Client + +Client -> MasterService: PutEnd(key, metadata) +activate MasterService +MasterService -> MasterService: 更新本地metadata +MasterService -> MasterService: SerializeMetadataForOpLog() +MasterService -> OpLogManager: Append(PUT_END, key, payload) +activate OpLogManager +OpLogManager -> OpLogManager: 生成sequence_id +OpLogManager -> OpLogManager: 计算checksum和prefix_hash +OpLogManager -> EtcdOpLogStore: WriteOpLog(entry) +activate EtcdOpLogStore +EtcdOpLogStore -> EtcdHelper: Put(key, value) +EtcdHelper -> etcd: 写入OpLog +etcd --> EtcdHelper: 成功 +EtcdHelper --> EtcdOpLogStore: 成功 +EtcdOpLogStore -> EtcdOpLogStore: TriggerBatchUpdateIfNeeded() +deactivate EtcdOpLogStore +OpLogManager --> MasterService: sequence_id +MasterService --> Client: 成功 +deactivate MasterService +deactivate OpLogManager + +' Standby接收OpLog +etcd -> OpLogWatcher: Watch事件(PUT) +activate OpLogWatcher +OpLogWatcher -> OpLogWatcher: DeserializeOpLogEntry() +OpLogWatcher -> OpLogApplier: ApplyOpLogEntry(entry) +activate OpLogApplier +OpLogApplier -> OpLogApplier: CheckSequenceOrder() +alt 顺序正确 + OpLogApplier -> OpLogApplier: ApplyPutEnd() + OpLogApplier -> StandbyMetadataStore: PutMetadata(key, metadata) + activate StandbyMetadataStore + StandbyMetadataStore --> OpLogApplier: 成功 + deactivate StandbyMetadataStore + OpLogApplier --> OpLogWatcher: 成功 +else 顺序错误(乱序) + OpLogApplier -> OpLogApplier: 加入pending队列 + OpLogApplier -> OpLogApplier: RequestMissingOpLog() + OpLogApplier -> EtcdOpLogStore: ReadOpLog(missing_seq_id) + activate EtcdOpLogStore + EtcdOpLogStore -> EtcdHelper: Get(key) + EtcdHelper -> etcd: 查询OpLog + etcd --> EtcdHelper: OpLog entry + EtcdHelper --> EtcdOpLogStore: entry + EtcdOpLogStore --> OpLogApplier: entry + deactivate EtcdOpLogStore + OpLogApplier -> OpLogApplier: ProcessPendingEntries() +end +deactivate OpLogApplier +deactivate OpLogWatcher + +== 故障切换流程 == + +participant NewPrimary as "New Primary\n(MasterService)" + +etcd -> ViewHelper: Leader lease过期 +activate ViewHelper +ViewHelper -> Supervisor: Leader已删除 +activate Supervisor + +Supervisor -> HotStandbyService: Promote() +activate HotStandbyService +HotStandbyService -> OpLogWatcher: Stop() +activate OpLogWatcher +OpLogWatcher --> HotStandbyService: 已停止 +deactivate OpLogWatcher + +HotStandbyService -> EtcdOpLogStore: ReadOpLogSince(last_seq_id) +activate EtcdOpLogStore +EtcdOpLogStore -> EtcdHelper: GetRangeAsJson() +EtcdHelper -> etcd: Range Get +etcd --> EtcdHelper: 剩余OpLog entries +EtcdHelper --> EtcdOpLogStore: entries +EtcdOpLogStore --> HotStandbyService: entries +deactivate EtcdOpLogStore + +loop 最终同步 + HotStandbyService -> OpLogApplier: ApplyOpLogEntries(batch) + activate OpLogApplier + OpLogApplier -> StandbyMetadataStore: 应用OpLog + StandbyMetadataStore --> OpLogApplier: 成功 + OpLogApplier --> HotStandbyService: applied_count + deactivate OpLogApplier +end + +HotStandbyService -> HotStandbyService: ExportMetadataSnapshot() +HotStandbyService -> HotStandbyService: GetLatestAppliedSequenceId() +HotStandbyService --> Supervisor: snapshot + last_seq_id +deactivate HotStandbyService + +Supervisor -> ViewHelper: ElectLeader() [重新选举] +activate ViewHelper +ViewHelper -> EtcdHelper: GrantLease() + CreateWithLease() +EtcdHelper -> etcd: 选举Leader +etcd --> EtcdHelper: 选举成功 +EtcdHelper --> ViewHelper: 成为新Leader +ViewHelper --> Supervisor: 选举成功 +deactivate ViewHelper + +Supervisor -> NewPrimary: 创建MasterService +activate NewPrimary +NewPrimary -> OpLogManager: SetInitialSequenceId(last_seq_id) +activate OpLogManager +OpLogManager --> NewPrimary: 已设置 +deactivate OpLogManager +NewPrimary -> NewPrimary: RestoreFromStandbySnapshot(snapshot) +NewPrimary -> NewPrimary: 恢复metadata到本地 +NewPrimary --> Supervisor: 恢复完成 +deactivate NewPrimary + +Supervisor -> NewPrimary: 启动服务 +activate NewPrimary +NewPrimary -> ViewHelper: KeepLeader(lease_id) +activate ViewHelper +ViewHelper -> EtcdHelper: KeepAlive(lease_id) +deactivate ViewHelper +deactivate NewPrimary +deactivate Supervisor + +@enduml + diff --git a/mooncake-common/etcd/etcd_wrapper.go b/mooncake-common/etcd/etcd_wrapper.go index 3d579c6d97..dd2ba7c674 100644 --- a/mooncake-common/etcd/etcd_wrapper.go +++ b/mooncake-common/etcd/etcd_wrapper.go @@ -9,6 +9,7 @@ import "C" import ( "context" + "encoding/json" "strings" "sync" "time" @@ -492,6 +493,56 @@ func EtcdStoreGetWithPrefixWrapper(prefix *C.char, prefixSize C.int, keys **C.ch return 0 } +//export EtcdStoreGetRangeAsJsonWrapper +func EtcdStoreGetRangeAsJsonWrapper(startKey *C.char, startKeySize C.int, endKey *C.char, endKeySize C.int, limit C.int, outJson **C.char, outJsonSize *C.int, revisionId *C.longlong, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + start := C.GoStringN(startKey, startKeySize) + end := C.GoStringN(endKey, endKeySize) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + opts := []clientv3.OpOption{ + clientv3.WithRange(end), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), + } + if limit > 0 { + opts = append(opts, clientv3.WithLimit(int64(limit))) + } + resp, err := storeClient.Get(ctx, start, opts...) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + if resp != nil && resp.Header != nil { + *revisionId = C.longlong(resp.Header.Revision) + } else { + *revisionId = 0 + } + + type kvPair struct { + Key string `json:"key"` + Value string `json:"value"` + } + kvs := make([]kvPair, 0, len(resp.Kvs)) + for _, kv := range resp.Kvs { + kvs = append(kvs, kvPair{Key: string(kv.Key), Value: string(kv.Value)}) + } + b, jerr := json.Marshal(kvs) + if jerr != nil { + *errMsg = C.CString(jerr.Error()) + return -1 + } + + *outJson = C.CString(string(b)) + *outJsonSize = C.int(len(b)) + return 0 +} + //export EtcdStoreGetFirstKeyWithPrefixWrapper func EtcdStoreGetFirstKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, firstKey **C.char, firstKeySize *C.int, errMsg **C.char) int { if storeClient == nil { @@ -620,6 +671,192 @@ func EtcdStoreWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, callbackC return 0 } +//export EtcdStoreWatchWithPrefixFromRevisionWrapper +func EtcdStoreWatchWithPrefixFromRevisionWrapper(prefix *C.char, prefixSize C.int, startRevision C.longlong, callbackContext unsafe.Pointer, callbackFunc unsafe.Pointer, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + if callbackFunc == nil { + *errMsg = C.CString("callback function is nil") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storePrefixWatchMutex.Lock() + if _, exists := storePrefixWatchCtx[p]; exists { + storePrefixWatchMutex.Unlock() + *errMsg = C.CString("This prefix is already being watched") + return -1 + } + storePrefixWatchCtx[p] = cancel + storePrefixWatchMutex.Unlock() + + go func() { + defer cancelAndDeletePrefixWatch(p) + + opts := []clientv3.OpOption{clientv3.WithPrefix()} + if startRevision > 0 { + opts = append(opts, clientv3.WithRev(int64(startRevision))) + } + watchChan := storeClient.Watch(ctx, p, opts...) + + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + return + } + if watchResp.Err() != nil { + return + } + + for _, event := range watchResp.Events { + keyStr := string(event.Kv.Key) + keyPtr := C.CString(keyStr) + keySize := C.size_t(len(keyStr)) + + var valuePtr *C.char + var valueSize C.size_t + var eventType C.int + + if event.Type == clientv3.EventTypePut { + eventType = C.int(0) + valueStr := string(event.Kv.Value) + valuePtr = C.CString(valueStr) + valueSize = C.size_t(len(valueStr)) + } else if event.Type == clientv3.EventTypeDelete { + eventType = C.int(1) + valuePtr = nil + valueSize = 0 + } + + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int))(callbackFunc) + (*callbackType)(callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType) + + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + } + case <-ctx.Done(): + return + } + } + }() + + return 0 +} + +//export EtcdStoreWatchWithPrefixFromRevisionV2Wrapper +func EtcdStoreWatchWithPrefixFromRevisionV2Wrapper(prefix *C.char, prefixSize C.int, startRevision C.longlong, callbackContext unsafe.Pointer, callbackFunc unsafe.Pointer, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + if callbackFunc == nil { + *errMsg = C.CString("callback function is nil") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + + ctx, cancel := context.WithCancel(context.Background()) + + storePrefixWatchMutex.Lock() + if _, exists := storePrefixWatchCtx[p]; exists { + storePrefixWatchMutex.Unlock() + *errMsg = C.CString("This prefix is already being watched") + return -1 + } + storePrefixWatchCtx[p] = cancel + storePrefixWatchMutex.Unlock() + + go func() { + defer cancelAndDeletePrefixWatch(p) + + opts := []clientv3.OpOption{clientv3.WithPrefix()} + if startRevision > 0 { + opts = append(opts, clientv3.WithRev(int64(startRevision))) + } + watchChan := storeClient.Watch(ctx, p, opts...) + + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + // Channel closed unexpectedly. Notify C++ watcher to reconnect. + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int, C.longlong))(callbackFunc) + (*callbackType)(callbackContext, nil, 0, nil, 0, C.int(2) /*WATCH_BROKEN*/, C.longlong(0)) + return + } + if watchResp.Err() != nil { + // Watch error, stop watching. Notify C++ watcher to reconnect. + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int, C.longlong))(callbackFunc) + (*callbackType)(callbackContext, nil, 0, nil, 0, C.int(2) /*WATCH_BROKEN*/, C.longlong(0)) + return + } + + // Use response-level revision as a more stable resume point. + // (It can be >= individual event's ModRevision.) + respRev := int64(0) + if watchResp.Header != nil { + respRev = watchResp.Header.Revision + } + + for _, event := range watchResp.Events { + keyStr := string(event.Kv.Key) + keyPtr := C.CString(keyStr) + keySize := C.size_t(len(keyStr)) + + var valuePtr *C.char + var valueSize C.size_t + var eventType C.int + + if event.Type == clientv3.EventTypePut { + eventType = C.int(0) + valueStr := string(event.Kv.Value) + valuePtr = C.CString(valueStr) + valueSize = C.size_t(len(valueStr)) + } else if event.Type == clientv3.EventTypeDelete { + eventType = C.int(1) + valuePtr = nil + valueSize = 0 + } + + modRev := C.longlong(0) + if event.Kv != nil { + evRev := event.Kv.ModRevision + if respRev > evRev { + evRev = respRev + } + modRev = C.longlong(evRev) + } else if respRev > 0 { + modRev = C.longlong(respRev) + } + + // Callback signature: + // void cb(void* ctx, char* key, size_t keySize, char* value, size_t valueSize, int eventType, long long modRev) + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int, C.longlong))(callbackFunc) + (*callbackType)(callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType, modRev) + + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + } + case <-ctx.Done(): + return + } + } + }() + + return 0 +} + func cancelAndDeletePrefixWatch(p string) int { storePrefixWatchMutex.Lock() defer storePrefixWatchMutex.Unlock() diff --git a/mooncake-store/include/etcd_helper.h b/mooncake-store/include/etcd_helper.h index 11e783ba54..e9a6044a76 100644 --- a/mooncake-store/include/etcd_helper.h +++ b/mooncake-store/include/etcd_helper.h @@ -115,6 +115,25 @@ class EtcdHelper { std::vector& keys, std::vector& values); + /* + * @brief Range get in etcd and return result as a JSON array string. + * This avoids complex cross-language memory management for key/value arrays. + * @param start_key: Start key (inclusive). + * @param start_key_size: Size in bytes. + * @param end_key: End key (exclusive). + * @param end_key_size: Size in bytes. + * @param limit: Maximum number of kvs to return (0 means no limit). + * @param json: Output JSON string, format: [{"key":"...","value":"..."}] + * @param revision_id: Output etcd revision of this read (resp.Header.Revision). + */ + static ErrorCode GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id); + /* * @brief Get the first key with a given prefix (sorted by key). * @param prefix: The prefix to search for. @@ -157,6 +176,31 @@ class EtcdHelper { void (*callback_func)(void*, const char*, size_t, const char*, size_t, int)); + /* + * @brief Watch all keys with a given prefix from a specific etcd revision. + * This is used to close the "read historical -> start watch" gap. + * @param start_revision: Watch events with revision >= start_revision (0 means from now). + */ + static ErrorCode WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int)); + + /* + * @brief Watch all keys with a given prefix from a specific etcd revision (V2). + * V2 callback includes `mod_revision` for precise resume. + * (Implementation may pass max(event.ModRevision, watchResp.Header.Revision).) + * @param callback_func: void cb(void* ctx, const char* key, size_t key_size, + * const char* value, size_t value_size, + * int event_type, int64_t mod_revision) + * event_type: 0=PUT, 1=DELETE, 2=WATCH_BROKEN (watch ended; reconnect) + */ + static ErrorCode WatchWithPrefixFromRevisionV2( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)); + /* * @brief Cancel watching a prefix. * @param prefix: The prefix to stop watching. diff --git a/mooncake-store/include/etcd_oplog_store.h b/mooncake-store/include/etcd_oplog_store.h index 6104976eec..55efa78d9d 100644 --- a/mooncake-store/include/etcd_oplog_store.h +++ b/mooncake-store/include/etcd_oplog_store.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -28,8 +29,12 @@ class EtcdOpLogStore { /** * @brief Constructor. * @param cluster_id: The cluster ID for this OpLog store. + * @param enable_latest_seq_batch_update: Whether to start background thread + * to batch-update `/latest`. Readers (Standby) should set this to false + * to avoid unnecessary thread creation. */ - explicit EtcdOpLogStore(const std::string& cluster_id); + explicit EtcdOpLogStore(const std::string& cluster_id, + bool enable_latest_seq_batch_update = false); /** * @brief Write an OpLog entry to etcd. @@ -56,6 +61,12 @@ class EtcdOpLogStore { ErrorCode ReadOpLogSince(uint64_t start_sequence_id, size_t limit, std::vector& entries); + // Like ReadOpLogSince, but also returns the etcd revision for consistent + // "read then watch(from revision+1)" startup. + ErrorCode ReadOpLogSinceWithRevision(uint64_t start_sequence_id, size_t limit, + std::vector& entries, + EtcdRevisionId& revision_id); + /** * @brief Get the latest sequence_id from etcd. * @param sequence_id: Output param, the latest sequence_id. @@ -122,6 +133,11 @@ class EtcdOpLogStore { */ std::string BuildSnapshotKey(const std::string& snapshot_id) const; + // Best-effort: find the minimum existing OpLog sequence_id in etcd. + // Used for robust cleanup (Scheme 3) so we don't rely on a persisted + // "cleaned_upto" marker. + std::optional GetMinSequenceId() const; + /** * @brief Serialize an OpLogEntry to JSON string. * @param entry: The OpLog entry to serialize. @@ -161,6 +177,7 @@ class EtcdOpLogStore { static constexpr const char* kSnapshotSuffix = "/snapshot/"; // Batch update mechanism for latest_sequence_id + const bool enable_latest_seq_batch_update_{false}; std::atomic pending_latest_seq_id_{0}; std::atomic pending_count_{0}; std::atomic batch_update_running_{false}; diff --git a/mooncake-store/include/hot_standby_service.h b/mooncake-store/include/hot_standby_service.h index 3de61928be..7b7752d66a 100644 --- a/mooncake-store/include/hot_standby_service.h +++ b/mooncake-store/include/hot_standby_service.h @@ -8,11 +8,14 @@ #include #include #include +#include +#include #include "metadata_store.h" #include "oplog_applier.h" #include "oplog_manager.h" #include "oplog_watcher.h" +#include "snapshot_provider.h" #include "types.h" namespace mooncake { @@ -31,6 +34,11 @@ struct HotStandbyConfig { uint32_t verification_interval_sec{30}; uint32_t max_replication_lag_entries{1000}; bool enable_verification{true}; + + // Snapshot bootstrap (optional): + // If provided, Standby will try to load a snapshot first, then replay OpLog + // from snapshot_sequence_id. + bool enable_snapshot_bootstrap{false}; }; /** @@ -116,6 +124,15 @@ class HotStandbyService { */ uint64_t GetLatestAppliedSequenceId() const; + // Export a point-in-time snapshot of all replicated metadata. + // This is used by MasterServiceSupervisor to initialize the new Primary + // after leader election (fast recovery). + bool ExportMetadataSnapshot( + std::vector>& out) const; + + // Inject a snapshot provider (from external snapshot implementation). + void SetSnapshotProvider(std::unique_ptr provider); + private: /** * @brief Main replication loop (runs in background thread) @@ -165,11 +182,16 @@ class HotStandbyService { bool Exists(const std::string& key) const override; size_t GetKeyCount() const override; + // Snapshot for promotion/restore. + void Snapshot( + std::vector>& out) const; + private: mutable std::mutex mutex_; std::unordered_map store_; }; std::unique_ptr metadata_store_; + std::unique_ptr snapshot_provider_{std::make_unique()}; // OpLog replication components std::unique_ptr oplog_applier_; diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index b8b65cd6ba..4c631e8d21 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -26,11 +27,14 @@ #include "rpc_types.h" #include "replica.h" #include "oplog_manager.h" +#include "metadata_store.h" namespace mooncake { // Forward declarations class AllocationStrategy; class EvictionStrategy; +class BufferAllocatorBase; +struct StandbyObjectMetadata; // ReplicationService forward declaration removed - using etcd-based OpLog sync instead /* @@ -96,6 +100,13 @@ class MasterService { */ auto GetAllKeys() -> tl::expected, ErrorCode>; + // Restore metadata from a Standby snapshot (fast failover). + // NOTE: This is used only on the node that was running HotStandbyService + // right before it was promoted to leader. + void RestoreFromStandbySnapshot( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id); + /** * @brief Fetch all segments, each node has a unique real client with fixed * segment name : segment name, preferred format : {ip}:{port}, bad format : @@ -658,6 +669,13 @@ class MasterService { SegmentManager segment_manager_; BufferAllocatorType memory_allocator_type_; + // Keep dummy allocators alive for memory replicas restored from standby. + // AllocatedBuffer stores allocator as weak_ptr; without an owning shared_ptr, + // the allocator would expire immediately and transport_endpoint_ would be lost + // when re-serializing Replica descriptors. + std::unordered_map> + standby_allocator_keepalive_; + // Operation log manager for hot-standby replication. It records // state-changing operations so that a standby master can replay them. OpLogManager oplog_manager_; diff --git a/mooncake-store/include/metadata_store.h b/mooncake-store/include/metadata_store.h index 308fe1a636..33842238f4 100644 --- a/mooncake-store/include/metadata_store.h +++ b/mooncake-store/include/metadata_store.h @@ -22,8 +22,9 @@ struct StandbyObjectMetadata { UUID client_id{0, 0}; uint64_t size{0}; std::vector replicas; - uint64_t lease_timeout_ms{0}; // Lease timeout as milliseconds since epoch - std::optional soft_pin_timeout_ms; // Soft pin timeout as milliseconds since epoch + // NOTE: Lease information is NOT stored because: + // 1. Standby does not perform eviction, so lease info is not used + // 2. After promotion, new Primary should grant fresh leases, not restore old ones uint64_t last_sequence_id{0}; // Last OpLog sequence ID that modified this key StandbyObjectMetadata() = default; @@ -42,11 +43,9 @@ struct MetadataPayload { uint64_t client_id_second{0}; // UUID.second uint64_t size{0}; std::vector replicas; - uint64_t lease_timeout_ms{0}; - std::optional soft_pin_timeout_ms; + // NOTE: Lease information removed - not needed by Standby - YLT_REFL(MetadataPayload, client_id_first, client_id_second, size, replicas, - lease_timeout_ms, soft_pin_timeout_ms); + YLT_REFL(MetadataPayload, client_id_first, client_id_second, size, replicas); // Convert to StandbyObjectMetadata StandbyObjectMetadata ToStandbyMetadata(uint64_t sequence_id) const { @@ -54,8 +53,6 @@ struct MetadataPayload { meta.client_id = {client_id_first, client_id_second}; meta.size = size; meta.replicas = replicas; - meta.lease_timeout_ms = lease_timeout_ms; - meta.soft_pin_timeout_ms = soft_pin_timeout_ms; meta.last_sequence_id = sequence_id; return meta; } diff --git a/mooncake-store/include/oplog_watcher.h b/mooncake-store/include/oplog_watcher.h index 35a9f68e00..f0be684fd2 100644 --- a/mooncake-store/include/oplog_watcher.h +++ b/mooncake-store/include/oplog_watcher.h @@ -8,6 +8,7 @@ #include #include "oplog_manager.h" +#include "types.h" namespace mooncake { @@ -38,6 +39,14 @@ class OpLogWatcher { */ void Start(); + /** + * @brief Start from a known last-applied sequence_id. + * + * It will read historical OpLogs at a consistent etcd revision, then start + * watch from revision+1 to close the gap between "read" and "watch". + */ + bool StartFromSequenceId(uint64_t start_seq_id); + /** * @brief Stop watching */ @@ -59,6 +68,9 @@ class OpLogWatcher { uint64_t GetLastProcessedSequenceId() const; private: + bool ReadOpLogSinceWithRevision(uint64_t start_seq_id, + std::vector& entries, + EtcdRevisionId& revision_id); /** * @brief Static callback function for etcd Watch * @param context OpLogWatcher instance (passed as void*) @@ -71,6 +83,11 @@ class OpLogWatcher { static void WatchCallback(void* context, const char* key, size_t key_size, const char* value, size_t value_size, int event_type); + // V2 callback includes etcd KV mod_revision for precise resume. + static void WatchCallbackV2(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, int event_type, + int64_t mod_revision); + /** * @brief Watch etcd OpLog changes (runs in background thread) */ @@ -84,6 +101,8 @@ class OpLogWatcher { */ void HandleWatchEvent(const std::string& key, const std::string& value, int event_type); + void HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision); /** * @brief Deserialize OpLogEntry from JSON string @@ -104,6 +123,9 @@ class OpLogWatcher { */ bool SyncMissedEntries(); + // Next watch revision (0 means from now). Updated by consistent reads. + std::atomic next_watch_revision_{0}; + std::string etcd_endpoints_; std::string cluster_id_; OpLogApplier* applier_; diff --git a/mooncake-store/include/rpc_service.h b/mooncake-store/include/rpc_service.h index 7e0aef3a7d..465a6d5661 100644 --- a/mooncake-store/include/rpc_service.h +++ b/mooncake-store/include/rpc_service.h @@ -4,11 +4,13 @@ #include #include #include +#include #include #include #include #include "master_service.h" +#include "metadata_store.h" #include "types.h" #include "rpc_types.h" #include "master_config.h" @@ -30,6 +32,11 @@ class WrappedMasterService { void init_http_server(); + // Restore metadata and OpLog sequence from a promoted Standby (fast failover). + void RestoreFromStandby( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id); + tl::expected ExistKey(const std::string& key); tl::expected diff --git a/mooncake-store/include/snapshot_provider.h b/mooncake-store/include/snapshot_provider.h new file mode 100644 index 0000000000..ae8041511c --- /dev/null +++ b/mooncake-store/include/snapshot_provider.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +#include "metadata_store.h" + +namespace mooncake { + +/** + * @brief SnapshotProvider is an abstraction for loading metadata snapshots. + * + * Assumption: snapshot functionality exists (implemented by another team), but + * may not be synced into this repo yet. We keep Mooncake-store code progressing + * by depending on this narrow interface. + * + * Snapshot semantics for hot-standby: + * - A snapshot represents a consistent metadata baseline at `snapshot_sequence_id`. + * - Standby should: load snapshot -> recover applier to snapshot_sequence_id -> + * replay OpLog entries with sequence_id > snapshot_sequence_id. + */ +class SnapshotProvider { + public: + virtual ~SnapshotProvider() = default; + + // Load the latest available snapshot for `cluster_id`. + // Returns true on success and fills: + // - snapshot_id: opaque identifier (e.g. timestamp/version) + // - snapshot_sequence_id: global OpLog sequence_id at snapshot boundary + // - snapshot: full metadata baseline as key -> StandbyObjectMetadata + virtual bool LoadLatestSnapshot( + const std::string& cluster_id, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) = 0; +}; + +// Default no-op provider: behaves as if "no snapshot available". +class NoopSnapshotProvider final : public SnapshotProvider { + public: + bool LoadLatestSnapshot( + const std::string& /*cluster_id*/, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) override { + snapshot_id.clear(); + snapshot_sequence_id = 0; + snapshot.clear(); + return false; + } +}; + +} // namespace mooncake + + diff --git a/mooncake-store/src/etcd_helper.cpp b/mooncake-store/src/etcd_helper.cpp index 88c31d0fc2..b65a6dcafd 100644 --- a/mooncake-store/src/etcd_helper.cpp +++ b/mooncake-store/src/etcd_helper.cpp @@ -176,6 +176,33 @@ ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size return ErrorCode::INTERNAL_ERROR; } +ErrorCode EtcdHelper::GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id) { + char* err_msg = nullptr; + char* json_ptr = nullptr; + int json_size = 0; + // Go wrapper takes int limit. + int ret = EtcdStoreGetRangeAsJsonWrapper((char*)start_key, (int)start_key_size, + (char*)end_key, (int)end_key_size, + (int)limit, &json_ptr, &json_size, + (GoInt64*)&revision_id, &err_msg); + if (ret != 0) { + LOG(ERROR) << "start_key=" << std::string(start_key, start_key_size) + << ", end_key=" << std::string(end_key, end_key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + json = std::string(json_ptr, json_size); + free(json_ptr); + return ErrorCode::OK; +} + ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, const size_t prefix_size, std::string& first_key) { @@ -238,6 +265,59 @@ ErrorCode EtcdHelper::WatchWithPrefix(const char* prefix, const size_t prefix_si return ErrorCode::OK; } +ErrorCode EtcdHelper::WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int)) { + char* err_msg = nullptr; + void* callback_func_ptr = reinterpret_cast(callback_func); + int ret = EtcdStoreWatchWithPrefixFromRevisionWrapper( + (char*)prefix, (int)prefix_size, (GoInt64)start_revision, callback_context, + callback_func_ptr, &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", start_revision=" << (int64_t)start_revision + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::WatchWithPrefixFromRevisionV2( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)) { + char* err_msg = nullptr; + void* callback_func_ptr = reinterpret_cast(callback_func); + int ret = EtcdStoreWatchWithPrefixFromRevisionV2Wrapper( + (char*)prefix, (int)prefix_size, (GoInt64)start_revision, callback_context, + callback_func_ptr, &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", start_revision=" << (int64_t)start_revision + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::WatchWithPrefixFromRevisionV2( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)) { + (void)prefix; + (void)prefix_size; + (void)start_revision; + (void)callback_context; + (void)callback_func; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, const size_t prefix_size) { char* err_msg = nullptr; @@ -312,6 +392,23 @@ ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size return ErrorCode::ETCD_OPERATION_ERROR; } +ErrorCode EtcdHelper::GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id) { + (void)start_key; + (void)start_key_size; + (void)end_key; + (void)end_key_size; + (void)limit; + (void)json; + (void)revision_id; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, const size_t prefix_size, std::string& first_key) { @@ -335,6 +432,18 @@ ErrorCode EtcdHelper::WatchWithPrefix(const char* prefix, const size_t prefix_si return ErrorCode::ETCD_OPERATION_ERROR; } +ErrorCode EtcdHelper::WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int)) { + (void)prefix; + (void)prefix_size; + (void)start_revision; + (void)callback_context; + (void)callback_func; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, const size_t prefix_size) { LOG(FATAL) << "Etcd is not enabled in compilation"; diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp index de200715e4..2ad5ab1757 100644 --- a/mooncake-store/src/etcd_oplog_store.cpp +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -2,6 +2,7 @@ #include #include +#include #if __has_include() #include // Ubuntu @@ -13,21 +14,30 @@ namespace mooncake { -EtcdOpLogStore::EtcdOpLogStore(const std::string& cluster_id) +EtcdOpLogStore::EtcdOpLogStore(const std::string& cluster_id, + bool enable_latest_seq_batch_update) : cluster_id_(cluster_id), + enable_latest_seq_batch_update_(enable_latest_seq_batch_update), last_update_time_(std::chrono::steady_clock::now()) { - // Start batch update thread - batch_update_running_.store(true); - batch_update_thread_ = std::thread(&EtcdOpLogStore::BatchUpdateThread, this); + // Start batch update thread only for writers. + if (enable_latest_seq_batch_update_) { + batch_update_running_.store(true); + batch_update_thread_ = + std::thread(&EtcdOpLogStore::BatchUpdateThread, this); + } } EtcdOpLogStore::~EtcdOpLogStore() { + if (!enable_latest_seq_batch_update_) { + return; + } + // Stop batch update thread batch_update_running_.store(false); if (batch_update_thread_.joinable()) { batch_update_thread_.join(); } - + // Perform final update if there are pending updates if (pending_count_.load() > 0) { DoBatchUpdate(); @@ -46,11 +56,15 @@ ErrorCode EtcdOpLogStore::WriteOpLog(const OpLogEntry& entry) { return err; } - // Add to batch update queue instead of immediate update + // Update `/latest`. + // - Writers: batch update to reduce etcd write pressure. + // - Readers / tests: update immediately for simplicity. + if (!enable_latest_seq_batch_update_) { + return UpdateLatestSequenceId(entry.sequence_id); + } + pending_latest_seq_id_.store(entry.sequence_id); size_t count = pending_count_.fetch_add(1) + 1; - - // Trigger immediate update if batch size threshold is reached if (count >= kBatchSize) { DoBatchUpdate(); } @@ -80,26 +94,113 @@ ErrorCode EtcdOpLogStore::ReadOpLog(uint64_t sequence_id, ErrorCode EtcdOpLogStore::ReadOpLogSince(uint64_t start_sequence_id, size_t limit, std::vector& entries) { - // TODO: Implement ReadOpLogSince using GetWithPrefix - // For now, read entries one by one (inefficient but works) + EtcdRevisionId rev = 0; + return ReadOpLogSinceWithRevision(start_sequence_id, limit, entries, rev); +} + +ErrorCode EtcdOpLogStore::ReadOpLogSinceWithRevision(uint64_t start_sequence_id, + size_t limit, + std::vector& entries, + EtcdRevisionId& revision_id) { entries.clear(); entries.reserve(limit); - uint64_t current_seq = start_sequence_id + 1; - for (size_t i = 0; i < limit; ++i) { - OpLogEntry entry; - ErrorCode err = ReadOpLog(current_seq, entry); - if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { - // No more entries - break; + // Range is limited to OpLog entry keys only. + const std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/"; + std::string current_start_key = BuildOpLogKey(start_sequence_id + 1); + + // Compute prefix range end (etcd prefix end). + auto prefix_end = [](std::string p) -> std::string { + for (int i = static_cast(p.size()) - 1; i >= 0; --i) { + unsigned char c = static_cast(p[i]); + if (c < 0xFF) { + p[i] = static_cast(c + 1); + p.resize(i + 1); + return p; + } } + return std::string(1, '\0'); + }; + const std::string end_key = prefix_end(prefix); + + // Pagination: + // - Use range-get with limit + // - Start next page from lastKey + '\0' (lexicographically just after lastKey) + // This avoids repeating the last key without adding new Go/C++ APIs. + revision_id = 0; + while (entries.size() < limit) { + const size_t page_limit = limit - entries.size(); + std::string json; + EtcdRevisionId page_rev = 0; + ErrorCode err = + EtcdHelper::GetRangeAsJson(current_start_key.c_str(), + current_start_key.size(), end_key.c_str(), + end_key.size(), page_limit, json, page_rev); if (err != ErrorCode::OK) { - LOG(ERROR) << "Failed to read OpLog entry, sequence_id=" - << current_seq; return err; } - entries.push_back(entry); - current_seq++; + if (page_rev > revision_id) { + revision_id = page_rev; + } + + // Parse kv list: [{"key":"...","value":"..."}] + Json::Value root; + Json::CharReaderBuilder reader; + std::string errs; + std::istringstream s(json); + if (!Json::parseFromStream(reader, s, &root, &errs)) { + LOG(ERROR) << "Failed to parse range JSON: " << errs; + return ErrorCode::INTERNAL_ERROR; + } + if (!root.isArray()) { + return ErrorCode::INTERNAL_ERROR; + } + if (root.empty()) { + break; // no more data + } + + std::string last_key_in_page; + for (const auto& kv : root) { + const std::string key = kv.get("key", "").asString(); + last_key_in_page = key; + if (key.empty() || key.find("/latest") != std::string::npos || + key.find("/snapshot/") != std::string::npos) { + continue; + } + + // Parse seq from key suffix and filter (handles legacy keys too). + size_t pos = key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= key.size()) { + continue; + } + uint64_t seq = 0; + try { + seq = static_cast(std::stoull(key.substr(pos + 1))); + } catch (...) { + continue; + } + if (seq <= start_sequence_id) { + continue; + } + + OpLogEntry entry; + const std::string value = kv.get("value", "").asString(); + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry from key=" << key; + return ErrorCode::INTERNAL_ERROR; + } + entries.push_back(std::move(entry)); + if (entries.size() >= limit) { + break; + } + } + + // Advance start key for next page. + if (last_key_in_page.empty()) { + break; + } + current_start_key = last_key_in_page; + current_start_key.push_back('\0'); } return ErrorCode::OK; @@ -158,9 +259,24 @@ ErrorCode EtcdOpLogStore::GetSnapshotSequenceId( } ErrorCode EtcdOpLogStore::CleanupOpLogBefore(uint64_t before_sequence_id) { - // Build start and end keys for the range - std::string start_key = BuildOpLogKey(1); // Start from sequence_id 1 - std::string end_key = BuildOpLogKey(before_sequence_id); // End before this + // Robust cleanup (Scheme 3): + // - Determine current minimum sequence_id in etcd + // - DeleteRange [min_key, before_key) + // + // IMPORTANT: This relies on lexicographical ordering of keys, so the + // sequence_id portion MUST be fixed-width (zero-padded). + auto min_seq_opt = GetMinSequenceId(); + if (!min_seq_opt.has_value()) { + return ErrorCode::OK; // nothing to cleanup + } + + uint64_t min_seq = min_seq_opt.value(); + if (before_sequence_id <= min_seq) { + return ErrorCode::OK; + } + + std::string start_key = BuildOpLogKey(min_seq); + std::string end_key = BuildOpLogKey(before_sequence_id); // delete < before_sequence_id return EtcdHelper::DeleteRange(start_key.c_str(), start_key.size(), end_key.c_str(), end_key.size()); @@ -168,10 +284,42 @@ ErrorCode EtcdOpLogStore::CleanupOpLogBefore(uint64_t before_sequence_id) { std::string EtcdOpLogStore::BuildOpLogKey(uint64_t sequence_id) const { std::ostringstream oss; - oss << kOpLogPrefix << cluster_id_ << "/" << sequence_id; + // Fixed-width encoding for correct etcd lexicographical range operations. + // 20 digits is enough for uint64_t max (18446744073709551615). + oss << kOpLogPrefix << cluster_id_ << "/" + << std::setw(20) << std::setfill('0') << sequence_id; return oss.str(); } +std::optional EtcdOpLogStore::GetMinSequenceId() const { + std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/"; + std::string first_key; + ErrorCode err = + EtcdHelper::GetFirstKeyWithPrefix(prefix.c_str(), prefix.size(), first_key); + if (err != ErrorCode::OK) { + return std::nullopt; + } + + // Skip non-entry keys if any (e.g. "/latest" or "/snapshot/..."). + // Entries are expected to be ".../<20-digit-seq>". + // If the first key isn't an entry key, fall back to nullopt (safe no-op). + if (first_key.find("/latest") != std::string::npos || + first_key.find("/snapshot/") != std::string::npos) { + return std::nullopt; + } + + size_t pos = first_key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= first_key.size()) { + return std::nullopt; + } + std::string seq_str = first_key.substr(pos + 1); + try { + return static_cast(std::stoull(seq_str)); + } catch (...) { + return std::nullopt; + } +} + std::string EtcdOpLogStore::BuildLatestKey() const { std::ostringstream oss; oss << kOpLogPrefix << cluster_id_ << kLatestSuffix; @@ -237,6 +385,9 @@ bool EtcdOpLogStore::DeserializeOpLogEntry(const std::string& json_str, } void EtcdOpLogStore::BatchUpdateThread() { + if (!enable_latest_seq_batch_update_) { + return; + } while (batch_update_running_.load()) { std::this_thread::sleep_for( std::chrono::milliseconds(kBatchIntervalMs)); @@ -262,6 +413,9 @@ void EtcdOpLogStore::TriggerBatchUpdateIfNeeded() { } void EtcdOpLogStore::DoBatchUpdate() { + if (!enable_latest_seq_batch_update_) { + return; + } std::lock_guard lock(batch_update_mutex_); // Get the pending sequence_id and reset counters diff --git a/mooncake-store/src/ha_helper.cpp b/mooncake-store/src/ha_helper.cpp index a7f79e9708..5a50cb4013 100644 --- a/mooncake-store/src/ha_helper.cpp +++ b/mooncake-store/src/ha_helper.cpp @@ -157,12 +157,14 @@ int MasterServiceSupervisor::Start() { std::string current_leader; ViewVersionId current_version = 0; auto ret = mv_helper.GetMasterView(current_leader, current_version); + bool had_standby = false; if (ret == ErrorCode::OK) { // There is an existing leader, start Standby service LOG(INFO) << "Found existing leader: " << current_leader << ", starting Standby service..."; StartStandbyService(mv_helper, current_leader); + had_standby = true; // Build master_view_key (same logic as MasterViewHelper) std::string cluster_id = config_.cluster_id; @@ -176,11 +178,10 @@ int MasterServiceSupervisor::Start() { auto watch_ret = EtcdHelper::WatchUntilDeleted( master_view_key.c_str(), master_view_key.size()); - // Stop Standby service when leader disappears - StopStandbyService(); - if (watch_ret != ErrorCode::OK) { LOG(ERROR) << "Error watching for leadership change: " << watch_ret; + // Stop Standby service on watch error and retry. + StopStandbyService(); std::this_thread::sleep_for(std::chrono::seconds(1)); continue; } @@ -193,6 +194,22 @@ int MasterServiceSupervisor::Start() { // Try to elect self as leader mv_helper.ElectLeader(config_.local_hostname, view_version, lease_id); + // If we were running as Standby, finalize catch-up and snapshot metadata now. + std::vector> standby_snapshot; + uint64_t standby_last_seq_id = 0; +#ifdef STORE_USE_ETCD + if (had_standby && standby_service_ && standby_running_.load()) { + LOG(INFO) << "Finalizing standby state for promotion..."; + standby_service_->Promote(); // does final catch-up sync + stops watcher + standby_last_seq_id = standby_service_->GetLatestAppliedSequenceId(); + standby_service_->ExportMetadataSnapshot(standby_snapshot); + // We are now leader; standby service is no longer needed. + StopStandbyService(); + LOG(INFO) << "Standby snapshot ready: keys=" << standby_snapshot.size() + << ", last_seq_id=" << standby_last_seq_id; + } +#endif + // Start a thread to keep the leader alive auto keep_leader_thread = std::thread([&server, &mv_helper, lease_id]() { @@ -209,6 +226,14 @@ int MasterServiceSupervisor::Start() { LOG(INFO) << "Starting master service..."; mooncake::WrappedMasterService wrapped_master_service( mooncake::WrappedMasterServiceConfig(config_, view_version)); + + // Restore from promoted standby snapshot if available. +#ifdef STORE_USE_ETCD + if (standby_last_seq_id > 0 || !standby_snapshot.empty()) { + wrapped_master_service.RestoreFromStandby(standby_snapshot, standby_last_seq_id); + } +#endif + mooncake::RegisterRpcService(server, wrapped_master_service); // Metric reporting is now handled by WrappedMasterService. diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index c0905783ae..fb5912fa9e 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -73,6 +73,16 @@ size_t HotStandbyService::StandbyMetadataStore::GetKeyCount() const { return store_.size(); } +void HotStandbyService::StandbyMetadataStore::Snapshot( + std::vector>& out) const { + std::lock_guard lock(mutex_); + out.clear(); + out.reserve(store_.size()); + for (const auto& kv : store_) { + out.emplace_back(kv.first, kv.second); + } +} + HotStandbyService::~HotStandbyService() { Stop(); } @@ -99,8 +109,27 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, return err; } - // Recreate OpLogApplier with cluster_id (for requesting missing OpLog) + // Preserve existing local state if HotStandbyService is restarted in-process: + // - metadata_store_ may already contain real-time metadata + // - oplog_applier_ may already have expected_sequence_id_ + uint64_t local_last_seq_id = 0; + if (oplog_applier_) { + uint64_t expected = oplog_applier_->GetExpectedSequenceId(); + local_last_seq_id = expected > 0 ? expected - 1 : 0; + } + const bool has_local_metadata = + metadata_store_ && metadata_store_->GetKeyCount() > 0; + const bool has_local_state = has_local_metadata && local_last_seq_id > 0; + + // Recreate OpLogApplier with cluster_id (for requesting missing OpLog). + // If we had local state, recover to keep sequence continuity. oplog_applier_ = std::make_unique(metadata_store_.get(), cluster_id); + if (has_local_state) { + LOG(INFO) << "Standby warm start: reuse local metadata (keys=" + << metadata_store_->GetKeyCount() + << "), recover last_seq_id=" << local_last_seq_id; + oplog_applier_->Recover(local_last_seq_id); + } // Create OpLogWatcher oplog_watcher_ = std::make_unique( @@ -109,28 +138,39 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, running_.store(true); is_connected_.store(true); - // Read historical OpLog entries first - // Get the last applied sequence ID from OpLogApplier - uint64_t last_applied_seq_id = oplog_applier_->GetExpectedSequenceId() - 1; - if (last_applied_seq_id == 0) { - // First time - start from sequence_id 0 (will read from sequence_id 1) - last_applied_seq_id = 0; + // Bootstrap: + // - If we already have local state (warm start), do NOT reload snapshot. + // - Otherwise (cold start/new standby), try snapshot (if enabled) then replay OpLog. + uint64_t baseline_seq_id = has_local_state ? local_last_seq_id : 0; + if (!has_local_state && config_.enable_snapshot_bootstrap && snapshot_provider_) { + std::string snapshot_id; + uint64_t snapshot_seq_id = 0; + std::vector> snapshot; + if (snapshot_provider_->LoadLatestSnapshot(cluster_id_, snapshot_id, snapshot_seq_id, + snapshot)) { + LOG(INFO) << "Loaded snapshot: snapshot_id=" << snapshot_id + << ", snapshot_seq_id=" << snapshot_seq_id + << ", keys=" << snapshot.size(); + // Apply snapshot into local standby store. + for (const auto& kv : snapshot) { + metadata_store_->PutMetadata(kv.first, kv.second); + } + // Align applier to snapshot boundary. + oplog_applier_->Recover(snapshot_seq_id); + baseline_seq_id = snapshot_seq_id; + } else { + LOG(INFO) << "No snapshot available (or provider not ready), falling back to OpLog-only bootstrap"; + } } - std::vector historical_entries; - if (oplog_watcher_->ReadOpLogSince(last_applied_seq_id, historical_entries)) { - LOG(INFO) << "Read " << historical_entries.size() - << " historical OpLog entries, applying..."; - // Apply historical entries - size_t applied_count = oplog_applier_->ApplyOpLogEntries(historical_entries); - LOG(INFO) << "Applied " << applied_count - << " historical OpLog entries"; - } else { - LOG(WARNING) << "Failed to read historical OpLog entries, continuing anyway"; - } + // Read historical OpLog entries since baseline_seq_id. + uint64_t last_applied_seq_id = baseline_seq_id; - // Start OpLogWatcher (this will start watching etcd in background) - oplog_watcher_->Start(); + // Start OpLogWatcher with a consistent "read then watch(from revision+1)" sequence. + if (!oplog_watcher_->StartFromSequenceId(last_applied_seq_id)) { + LOG(WARNING) << "Failed to start OpLogWatcher from sequence_id=" + << last_applied_seq_id << ", continuing anyway"; + } // Start background threads replication_thread_ = std::thread(&HotStandbyService::ReplicationLoop, this); @@ -186,9 +226,7 @@ StandbySyncStatus HotStandbyService::GetSyncStatus() const { status.applied_seq_id = applied_seq_id_.load(); } - // Get primary sequence ID from etcd (if OpLogWatcher is available) - // For now, we use a placeholder - in full implementation we would - // query etcd for the latest sequence_id + // Primary sequence ID (best-effort): updated by ReplicationLoop via etcd `/latest`. status.primary_seq_id = primary_seq_id_.load(); status.is_connected = is_connected_.load(); @@ -240,62 +278,36 @@ std::unique_ptr HotStandbyService::Promote() { << current_applied_seq_id << ", lag: " << status.lag_entries << " entries"; - // Continue syncing remaining OpLog entries from etcd before promotion - if (status.lag_entries > 0) { - LOG(INFO) << "Syncing remaining " << status.lag_entries - << " OpLog entries from etcd before promotion..."; - - // Get latest sequence_id from etcd - EtcdOpLogStore oplog_store(cluster_id_); - uint64_t latest_seq_id = 0; - ErrorCode err = oplog_store.GetLatestSequenceId(latest_seq_id); - if (err != ErrorCode::OK) { - LOG(WARNING) << "Failed to get latest sequence_id from etcd: " << err - << ". Will proceed with promotion, but metadata may be incomplete."; - } else { - // Read and apply remaining OpLog entries - uint64_t remaining_count = latest_seq_id - current_applied_seq_id; - if (remaining_count > 0) { - LOG(INFO) << "Reading " << remaining_count - << " remaining OpLog entries from etcd..."; - - std::vector remaining_entries; - // Read in batches to avoid memory issues - const size_t batch_size = 1000; - uint64_t start_seq = current_applied_seq_id + 1; - size_t total_applied = 0; - - while (start_seq <= latest_seq_id) { - std::vector batch; - ErrorCode read_err = oplog_store.ReadOpLogSince( - start_seq - 1, batch_size, batch); - - if (read_err != ErrorCode::OK) { - LOG(ERROR) << "Failed to read OpLog batch starting from " - << start_seq << ": " << read_err; - break; - } - - if (batch.empty()) { - break; // No more entries - } - - // Apply batch - size_t applied = oplog_applier_->ApplyOpLogEntries(batch); - total_applied += applied; - - LOG(INFO) << "Applied " << applied << " OpLog entries " - << "(batch: " << batch[0].sequence_id - << " to " << batch.back().sequence_id << ")"; - - start_seq = batch.back().sequence_id + 1; - } - - LOG(INFO) << "Completed syncing remaining OpLog entries. " - << "Total applied: " << total_applied; - } + // Final catch-up sync before promotion. + // IMPORTANT: + // - Do NOT rely on `lag_entries` here because primary_seq_id_ is best-effort. + // - Stop OpLogWatcher first to avoid concurrent Apply from watch callbacks. + if (oplog_watcher_) { + oplog_watcher_->Stop(); + } + + LOG(INFO) << "Final catch-up sync from etcd before promotion..."; + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + const size_t batch_size = 1000; + uint64_t start_seq = current_applied_seq_id + 1; + size_t total_applied = 0; + for (;;) { + std::vector batch; + ErrorCode read_err = oplog_store.ReadOpLogSince(start_seq - 1, batch_size, batch); + if (read_err != ErrorCode::OK) { + LOG(WARNING) << "Final catch-up: failed to read OpLog since seq=" + << (start_seq - 1) << ", err=" << read_err + << ". Proceeding with promotion."; + break; + } + if (batch.empty()) { + break; } + size_t applied = oplog_applier_->ApplyOpLogEntries(batch); + total_applied += applied; + start_seq = batch.back().sequence_id + 1; } + LOG(INFO) << "Final catch-up sync done. total_applied=" << total_applied; // Stop replication (OpLogWatcher will stop watching) Stop(); @@ -336,6 +348,26 @@ uint64_t HotStandbyService::GetLatestAppliedSequenceId() const { return applied_seq_id_.load(); } +bool HotStandbyService::ExportMetadataSnapshot( + std::vector>& out) const { + std::lock_guard lock(mutex_); + if (!metadata_store_) { + out.clear(); + return false; + } + metadata_store_->Snapshot(out); + return true; +} + +void HotStandbyService::SetSnapshotProvider(std::unique_ptr provider) { + std::lock_guard lock(mutex_); + if (provider) { + snapshot_provider_ = std::move(provider); + } else { + snapshot_provider_ = std::make_unique(); + } +} + void HotStandbyService::ReplicationLoop() { LOG(INFO) << "Replication loop started (etcd-based OpLog sync)"; @@ -358,8 +390,18 @@ void HotStandbyService::ReplicationLoop() { } } - // TODO: Update primary_seq_id by querying etcd for latest sequence_id - // For now, we assume it's being updated elsewhere + // Update primary_seq_id by querying etcd `/latest` (best-effort). + // Note: `/latest` is batch-updated on Primary, so this is for monitoring only. +#ifdef STORE_USE_ETCD + if (!cluster_id_.empty()) { + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + uint64_t latest_seq = 0; + ErrorCode err = oplog_store.GetLatestSequenceId(latest_seq); + if (err == ErrorCode::OK) { + primary_seq_id_.store(latest_seq); + } + } +#endif // Sleep and check again std::this_thread::sleep_for(std::chrono::milliseconds(1000)); diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index d0ad2541e4..602e393d1e 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -8,6 +8,7 @@ #include #include +#include "allocator.h" #include "etcd_helper.h" #include "etcd_oplog_store.h" #include "master_metric_manager.h" @@ -18,6 +19,63 @@ namespace mooncake { +namespace { + +// A minimal allocator implementation used only to keep AllocatedBuffer handles +// "valid" after standby promotion. It does NOT own memory. +class DummyBufferAllocator final : public BufferAllocatorBase { + public: + DummyBufferAllocator(std::string segment_name, std::string transport_endpoint) + : segment_name_(std::move(segment_name)), + transport_endpoint_(std::move(transport_endpoint)) {} + + std::unique_ptr allocate(size_t /*size*/) override { + return nullptr; + } + void deallocate(AllocatedBuffer* /*handle*/) override { + // no-op: we don't own memory + } + size_t capacity() const override { return kAllocatorUnknownFreeSpace; } + size_t size() const override { return 0; } + std::string getSegmentName() const override { return segment_name_; } + std::string getTransportEndpoint() const override { return transport_endpoint_; } + size_t getLargestFreeRegion() const override { return kAllocatorUnknownFreeSpace; } + + private: + std::string segment_name_; + std::string transport_endpoint_; +}; + +static Replica ReplicaFromDescriptor( + const Replica::Descriptor& desc, + const std::shared_ptr& allocator_keepalive) { + if (desc.is_memory_replica()) { + const auto& mem = desc.get_memory_descriptor(); + const auto& bd = mem.buffer_descriptor; + if (!allocator_keepalive) { + // This would make the buffer handle invalid immediately (allocator stored + // as weak_ptr in AllocatedBuffer). Callers restoring from standby should + // always provide a keepalive allocator. + LOG(ERROR) << "ReplicaFromDescriptor(memory) missing keepalive allocator, " + << "transport_endpoint=" << bd.transport_endpoint_; + } + + auto buf = std::make_unique( + allocator_keepalive, reinterpret_cast(bd.buffer_address_), + static_cast(bd.size_)); + return Replica(std::move(buf), desc.status); + } + if (desc.is_disk_replica()) { + const auto& disk = desc.get_disk_descriptor(); + return Replica(disk.file_path, disk.object_size, desc.status); + } + const auto& ld = desc.get_local_disk_descriptor(); + UUID client_id{ld.client_id_first, ld.client_id_second}; + return Replica(client_id, ld.object_size, ld.transport_endpoint, desc.status); +} + +} // namespace + MasterService::MasterService() : MasterService(MasterServiceConfig()) {} std::string MasterService::SerializeMetadataForOpLog(const ObjectMetadata& metadata) const { @@ -32,14 +90,9 @@ std::string MasterService::SerializeMetadataForOpLog(const ObjectMetadata& metad payload.replicas.push_back(replica.get_descriptor()); } - // Convert time_point to milliseconds since epoch - auto lease_duration = metadata.lease_timeout.time_since_epoch(); - payload.lease_timeout_ms = std::chrono::duration_cast(lease_duration).count(); - - if (metadata.soft_pin_timeout.has_value()) { - auto soft_pin_duration = metadata.soft_pin_timeout->time_since_epoch(); - payload.soft_pin_timeout_ms = std::chrono::duration_cast(soft_pin_duration).count(); - } + // NOTE: Lease information is NOT serialized because: + // 1. Standby does not perform eviction, so lease info is not used + // 2. After promotion, new Primary should grant fresh leases, not restore old ones // Serialize to JSON std::string json_str; @@ -113,8 +166,9 @@ MasterService::MasterService(const MasterServiceConfig& config) if (enable_ha_ && !cluster_id_.empty()) { // Try to create EtcdOpLogStore - if etcd is not connected, operations will fail // but we can still use memory buffer as fallback + // Writer: enable batch update for `/latest` to reduce etcd write pressure. auto etcd_oplog_store = - std::make_shared(cluster_id_); + std::make_shared(cluster_id_, /*enable_latest_seq_batch_update=*/true); oplog_manager_.SetEtcdOpLogStore(etcd_oplog_store); LOG(INFO) << "EtcdOpLogStore initialized for cluster_id=" << cluster_id_ << " (etcd connection should be established " @@ -141,6 +195,78 @@ void MasterService::AppendOpLogAndNotify(OpType type, const std::string& key, // TODO: In Phase 1, integrate with EtcdOpLogStore to write to etcd } +void MasterService::RestoreFromStandbySnapshot( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id) { + // 1) Ensure OpLog sequence continues without regression after failover. + oplog_manager_.SetInitialSequenceId(initial_oplog_sequence_id); + + // 2) Restore metadata entries. + // Keep dummy allocators alive for restored memory replicas. AllocatedBuffer + // only holds a weak_ptr to allocator, so without this keepalive map the + // allocator would expire immediately and transport_endpoint_ would be lost. + standby_allocator_keepalive_.clear(); + auto get_keepalive_allocator = + [this](const std::string& transport_endpoint) + -> std::shared_ptr { + auto it = standby_allocator_keepalive_.find(transport_endpoint); + if (it != standby_allocator_keepalive_.end()) { + return it->second; + } + auto alloc = std::make_shared( + /*segment_name=*/std::string(), transport_endpoint); + standby_allocator_keepalive_.emplace(transport_endpoint, alloc); + return alloc; + }; + + const auto now = std::chrono::steady_clock::now(); + size_t restored = 0; + for (const auto& kv : snapshot) { + const std::string& key = kv.first; + const StandbyObjectMetadata& sm = kv.second; + + std::vector replicas; + replicas.reserve(sm.replicas.size()); + for (const auto& rd : sm.replicas) { + if (rd.is_memory_replica()) { + const auto& bd = rd.get_memory_descriptor().buffer_descriptor; + replicas.emplace_back( + ReplicaFromDescriptor(rd, get_keepalive_allocator(bd.transport_endpoint_))); + } else { + replicas.emplace_back(ReplicaFromDescriptor(rd, nullptr)); + } + } + + // NOTE: Lease information is NOT restored because: + // 1. Standby does not use lease info (no eviction) + // 2. New Primary should grant fresh leases after promotion + // 3. Restoring old lease TTLs could cause immediate eviction if they're expired + const bool enable_soft_pin = false; // Will be set by new Primary if needed + + const size_t shard_idx = getShardIndex(key); + MutexLocker lock(&metadata_shards_[shard_idx].mutex); + + // Overwrite existing key if any. + metadata_shards_[shard_idx].metadata.erase(key); + auto [it, inserted] = metadata_shards_[shard_idx].metadata.emplace( + std::piecewise_construct, std::forward_as_tuple(key), + std::forward_as_tuple(sm.client_id, now, static_cast(sm.size), + std::move(replicas), enable_soft_pin)); + (void)inserted; + + // Lease will be granted by new Primary when objects are accessed + // (via GetReplicaList, ExistKey, etc.) + + // Objects restored from PUT_END are expected to be completed. + metadata_shards_[shard_idx].processing_keys.erase(key); + + restored++; + } + + LOG(INFO) << "Restored metadata from standby snapshot: restored_keys=" + << restored << ", initial_oplog_sequence_id=" << initial_oplog_sequence_id; +} + MasterService::~MasterService() { // Stop and join the threads eviction_running_ = false; diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index 5a8e10cad6..0029e6d6de 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -29,7 +29,9 @@ EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const { std::lock_guard lock(etcd_oplog_store_mutex_); if (!etcd_oplog_store_) { - etcd_oplog_store_ = std::make_unique(cluster_id_); + // Reader: do not start `/latest` batch update thread. + etcd_oplog_store_ = + std::make_unique(cluster_id_, /*enable_latest_seq_batch_update=*/false); } return etcd_oplog_store_.get(); #else @@ -155,61 +157,70 @@ size_t OpLogApplier::ProcessPendingEntries() { } } - // Now process pending entries - std::lock_guard lock(pending_mutex_); size_t processed_count = 0; + for (;;) { + OpLogEntry entry_copy; + bool has_entry = false; - // Process entries in order - while (!pending_entries_.empty()) { - auto it = pending_entries_.begin(); - const OpLogEntry& entry = it->second; + { + std::lock_guard lock(pending_mutex_); + if (pending_entries_.empty()) { + break; + } - // Check if this entry can be applied now - if (entry.sequence_id == expected_sequence_id_) { - // Release lock before applying (to avoid deadlock) - OpLogEntry entry_copy = entry; + auto it = pending_entries_.begin(); + if (it->first != expected_sequence_id_) { + break; // still waiting for earlier sequence_id + } + + entry_copy = it->second; pending_entries_.erase(it); + has_entry = true; + } - // Apply based on type - switch (entry_copy.op_type) { - case OpType::PUT_END: - ApplyPutEnd(entry_copy); - break; - case OpType::PUT_REVOKE: - ApplyPutRevoke(entry_copy); - break; - case OpType::REMOVE: - ApplyRemove(entry_copy); - break; - default: - LOG(ERROR) << "OpLogApplier: unsupported op_type in pending entry"; - continue; - } + if (!has_entry) { + break; + } - // Update expected sequence ID - expected_sequence_id_ = entry_copy.sequence_id + 1; + // Apply outside lock. + switch (entry_copy.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry_copy); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry_copy); + break; + case OpType::REMOVE: + ApplyRemove(entry_copy); + break; + default: + LOG(ERROR) << "OpLogApplier: unsupported op_type in pending entry"; + break; + } - // Remove from missing list if it was there - missing_sequence_ids_.erase(entry_copy.sequence_id); + expected_sequence_id_ = entry_copy.sequence_id + 1; - processed_count++; - } else { - // Cannot process more entries yet - break; + { + std::lock_guard lock(pending_mutex_); + missing_sequence_ids_.erase(entry_copy.sequence_id); } + + processed_count++; } // Clean up old missing sequence IDs (older than 1 minute) - auto now = std::chrono::steady_clock::now(); - for (auto it = missing_sequence_ids_.begin(); it != missing_sequence_ids_.end();) { - auto age = std::chrono::duration_cast(now - it->second); - if (age.count() > 60) { - // Too old, remove it - LOG(WARNING) << "OpLogApplier: giving up on missing sequence_id=" - << it->first << " after " << age.count() << " seconds"; - it = missing_sequence_ids_.erase(it); - } else { - ++it; + { + std::lock_guard lock(pending_mutex_); + auto now = std::chrono::steady_clock::now(); + for (auto it = missing_sequence_ids_.begin(); it != missing_sequence_ids_.end();) { + auto age = std::chrono::duration_cast(now - it->second); + if (age.count() > 60) { + LOG(WARNING) << "OpLogApplier: giving up on missing sequence_id=" + << it->first << " after " << age.count() << " seconds"; + it = missing_sequence_ids_.erase(it); + } else { + ++it; + } } } diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp index f5dbd43556..634fd5959b 100644 --- a/mooncake-store/src/oplog_watcher.cpp +++ b/mooncake-store/src/oplog_watcher.cpp @@ -32,14 +32,58 @@ OpLogWatcher::~OpLogWatcher() { } void OpLogWatcher::Start() { + // Backward-compatible: start from the last processed sequence id. + (void)StartFromSequenceId(last_processed_sequence_id_.load()); +} + +bool OpLogWatcher::StartFromSequenceId(uint64_t start_seq_id) { if (running_.load()) { LOG(WARNING) << "OpLogWatcher is already running"; - return; + return true; } +#ifdef STORE_USE_ETCD + uint64_t cursor_seq = start_seq_id; + EtcdRevisionId last_read_rev = 0; + size_t total_applied = 0; + + for (;;) { + std::vector batch; + EtcdRevisionId rev = 0; + if (!ReadOpLogSinceWithRevision(cursor_seq, batch, rev)) { + last_read_rev = 0; + break; + } + last_read_rev = rev; + if (!batch.empty()) { + for (const auto& e : batch) { + if (applier_->ApplyOpLogEntry(e)) { + last_processed_sequence_id_.store(e.sequence_id); + cursor_seq = e.sequence_id; + total_applied++; + } + } + } + if (batch.size() < kSyncBatchSize) { + break; + } + } + + if (last_read_rev > 0) { + next_watch_revision_.store(static_cast(last_read_rev + 1)); + } else { + next_watch_revision_.store(0); + } + + LOG(INFO) << "OpLogWatcher initial sync done: applied=" << total_applied + << ", last_seq=" << last_processed_sequence_id_.load() + << ", next_watch_revision=" << next_watch_revision_.load(); +#endif + running_.store(true); watch_thread_ = std::thread(&OpLogWatcher::WatchOpLog, this); LOG(INFO) << "OpLogWatcher started for cluster_id=" << cluster_id_; + return true; } void OpLogWatcher::Stop() { @@ -70,7 +114,7 @@ void OpLogWatcher::Stop() { bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, std::vector& entries) { #ifdef STORE_USE_ETCD - EtcdOpLogStore oplog_store(cluster_id_); + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); ErrorCode err = oplog_store.ReadOpLogSince(start_seq_id, 1000, entries); if (err != ErrorCode::OK) { LOG(ERROR) << "Failed to read OpLog since sequence_id=" << start_seq_id @@ -86,6 +130,27 @@ bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, #endif } +bool OpLogWatcher::ReadOpLogSinceWithRevision(uint64_t start_seq_id, + std::vector& entries, + EtcdRevisionId& revision_id) { +#ifdef STORE_USE_ETCD + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + ErrorCode err = oplog_store.ReadOpLogSinceWithRevision( + start_seq_id, kSyncBatchSize, entries, revision_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to read OpLog since sequence_id=" << start_seq_id + << ", error=" << static_cast(err); + return false; + } + return true; +#else + (void)start_seq_id; + (void)entries; + (void)revision_id; + return false; +#endif +} + uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { return last_processed_sequence_id_.load(); } @@ -99,13 +164,36 @@ void OpLogWatcher::WatchCallback(void* context, const char* key, size_t key_size return; } - std::string key_str(key, key_size); + std::string key_str; + if (key != nullptr && key_size > 0) { + key_str.assign(key, key_size); + } std::string value_str; if (value != nullptr && value_size > 0) { value_str = std::string(value, value_size); } - watcher->HandleWatchEvent(key_str, value_str, event_type); + watcher->HandleWatchEvent(key_str, value_str, event_type, /*mod_revision=*/0); +} + +void OpLogWatcher::WatchCallbackV2(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, + int event_type, int64_t mod_revision) { + OpLogWatcher* watcher = static_cast(context); + if (watcher == nullptr) { + LOG(ERROR) << "OpLogWatcher context is null"; + return; + } + + std::string key_str; + if (key != nullptr && key_size > 0) { + key_str.assign(key, key_size); + } + std::string value_str; + if (value != nullptr && value_size > 0) { + value_str = std::string(value, value_size); + } + watcher->HandleWatchEvent(key_str, value_str, event_type, mod_revision); } void OpLogWatcher::WatchOpLog() { @@ -116,8 +204,11 @@ void OpLogWatcher::WatchOpLog() { while (running_.load()) { // Start watching - pass static callback function and this pointer as context - ErrorCode err = EtcdHelper::WatchWithPrefix( - watch_prefix.c_str(), watch_prefix.size(), this, WatchCallback); + EtcdRevisionId start_rev = + static_cast(next_watch_revision_.load()); + // Always use V2 watcher so we can update next_watch_revision_ precisely. + ErrorCode err = EtcdHelper::WatchWithPrefixFromRevisionV2( + watch_prefix.c_str(), watch_prefix.size(), start_rev, this, WatchCallbackV2); if (err != ErrorCode::OK) { LOG(ERROR) << "Failed to start watch for prefix " << watch_prefix @@ -195,10 +286,14 @@ bool OpLogWatcher::SyncMissedEntries() { LOG(INFO) << "Syncing missed OpLog entries since sequence_id=" << last_seq; std::vector entries; - if (!ReadOpLogSince(last_seq, entries)) { + EtcdRevisionId rev = 0; + if (!ReadOpLogSinceWithRevision(last_seq, entries, rev)) { LOG(ERROR) << "Failed to read missed OpLog entries"; return false; } + if (rev > 0) { + next_watch_revision_.store(static_cast(rev + 1)); + } if (entries.empty()) { LOG(INFO) << "No missed OpLog entries to sync"; @@ -224,6 +319,31 @@ bool OpLogWatcher::SyncMissedEntries() { void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, int event_type) { + HandleWatchEvent(key, value, event_type, /*mod_revision=*/0); +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision) { + // event_type: + // 0 = PUT, 1 = DELETE, 2 = WATCH_BROKEN (Go watcher terminated; should reconnect) + if (event_type == 2) { + LOG(WARNING) << "OpLog watch broken, will reconnect. cluster_id=" << cluster_id_ + << ", next_watch_revision=" << next_watch_revision_.load() + << ", last_seq=" << last_processed_sequence_id_.load(); + watch_healthy_.store(false); + consecutive_errors_.fetch_add(1); + return; + } + + if (mod_revision > 0) { + // Keep next_watch_revision_ monotonic: next = max(next, modRev+1) + int64_t candidate = mod_revision + 1; + int64_t cur = next_watch_revision_.load(); + while (candidate > cur && + !next_watch_revision_.compare_exchange_weak(cur, candidate)) { + // retry + } + } // event_type: 0 = PUT, 1 = DELETE if (event_type == 1) { // DELETE event - OpLog entry was cleaned up @@ -311,6 +431,11 @@ void OpLogWatcher::Start() { LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; } +bool OpLogWatcher::StartFromSequenceId(uint64_t /*start_seq_id*/) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + void OpLogWatcher::Stop() { // No-op when STORE_USE_ETCD is not enabled } @@ -321,6 +446,13 @@ bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, return false; } +bool OpLogWatcher::ReadOpLogSinceWithRevision(uint64_t /*start_seq_id*/, + std::vector& /*entries*/, + EtcdRevisionId& /*revision_id*/) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { return last_processed_sequence_id_.load(); } @@ -334,6 +466,15 @@ void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& v LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; } +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision) { + (void)key; + (void)value; + (void)event_type; + (void)mod_revision; + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + void OpLogWatcher::TryReconnect() { LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; } diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index 6aa9cb86bd..261ffa330f 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -62,6 +62,12 @@ WrappedMasterService::~WrappedMasterService() { http_server_.stop(); } +void WrappedMasterService::RestoreFromStandby( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id) { + master_service_.RestoreFromStandbySnapshot(snapshot, initial_oplog_sequence_id); +} + void WrappedMasterService::init_http_server() { using namespace coro_http; From a483987f5473055457f3d964623b2dbd7f1dae1a Mon Sep 17 00:00:00 2001 From: BernardLee Date: Wed, 31 Dec 2025 15:47:10 +0800 Subject: [PATCH 35/59] fix --- mooncake-common/etcd/etcd_wrapper.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mooncake-common/etcd/etcd_wrapper.go b/mooncake-common/etcd/etcd_wrapper.go index dd2ba7c674..ff2f0060fe 100644 --- a/mooncake-common/etcd/etcd_wrapper.go +++ b/mooncake-common/etcd/etcd_wrapper.go @@ -802,8 +802,9 @@ func EtcdStoreWatchWithPrefixFromRevisionV2Wrapper(prefix *C.char, prefixSize C. // Use response-level revision as a more stable resume point. // (It can be >= individual event's ModRevision.) + // Note: watchResp.Header is a value type, not a pointer, so we can directly access it. respRev := int64(0) - if watchResp.Header != nil { + if watchResp.Header.Revision > 0 { respRev = watchResp.Header.Revision } From c1402e622be1e936512f901f4877e471b9e65cce Mon Sep 17 00:00:00 2001 From: BernardLee Date: Wed, 31 Dec 2025 15:49:41 +0800 Subject: [PATCH 36/59] fix --- mooncake-store/src/etcd_helper.cpp | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/mooncake-store/src/etcd_helper.cpp b/mooncake-store/src/etcd_helper.cpp index b65a6dcafd..df34f93945 100644 --- a/mooncake-store/src/etcd_helper.cpp +++ b/mooncake-store/src/etcd_helper.cpp @@ -304,20 +304,6 @@ ErrorCode EtcdHelper::WatchWithPrefixFromRevisionV2( return ErrorCode::OK; } -ErrorCode EtcdHelper::WatchWithPrefixFromRevisionV2( - const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, - void* callback_context, - void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, - int64_t)) { - (void)prefix; - (void)prefix_size; - (void)start_revision; - (void)callback_context; - (void)callback_func; - LOG(FATAL) << "Etcd is not enabled in compilation"; - return ErrorCode::ETCD_OPERATION_ERROR; -} - ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, const size_t prefix_size) { char* err_msg = nullptr; @@ -444,6 +430,21 @@ ErrorCode EtcdHelper::WatchWithPrefixFromRevision( LOG(FATAL) << "Etcd is not enabled in compilation"; return ErrorCode::ETCD_OPERATION_ERROR; } + +ErrorCode EtcdHelper::WatchWithPrefixFromRevisionV2( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)) { + (void)prefix; + (void)prefix_size; + (void)start_revision; + (void)callback_context; + (void)callback_func; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, const size_t prefix_size) { LOG(FATAL) << "Etcd is not enabled in compilation"; From a4a9512a654c16be64615a907772b50b11b047c8 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 5 Jan 2026 09:15:37 +0800 Subject: [PATCH 37/59] retry and seq miss process --- mooncake-store/include/master_service.h | 98 ++++ mooncake-store/include/oplog_applier.h | 26 +- mooncake-store/include/oplog_manager.h | 38 +- mooncake-store/src/etcd_oplog_store.cpp | 5 + mooncake-store/src/hot_standby_service.cpp | 11 + mooncake-store/src/master_service.cpp | 501 ++++++++++++++++++++- mooncake-store/src/oplog_applier.cpp | 215 +++++++-- mooncake-store/src/oplog_manager.cpp | 43 ++ mooncake-store/src/oplog_watcher.cpp | 18 +- 9 files changed, 898 insertions(+), 57 deletions(-) diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 4c631e8d21..7b42a3c3e1 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -4,7 +4,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -331,6 +334,12 @@ class MasterService { void AppendOpLogAndNotify(OpType type, const std::string& key, const std::string& payload = std::string()); + // Durable OpLog append: must succeed (write to etcd) for operations that may + // free/reuse memory (e.g. REMOVE). See OpLogManager::AppendAndPersist. + auto AppendOpLogAndNotifyDurable(OpType type, const std::string& key, + const std::string& payload = std::string()) + -> tl::expected; + // Resolve the key to a sanitized format for storage std::string SanitizeKey(const std::string& key) const; std::string ResolvePath(const std::string& key) const; @@ -514,6 +523,88 @@ class MasterService { */ std::string SerializeMetadataForOpLog(const ObjectMetadata& metadata) const; + // Serialize metadata but exclude MEMORY replicas. + // Used for eviction: when memory replicas are freed/reused, Standby must not + // keep stale memory descriptors. We persist a PUT_END containing only + // remaining (DISK/LOCAL_DISK) replicas before freeing memory. + std::string SerializeMetadataForOpLogWithoutMemReplicas( + const ObjectMetadata& metadata) const; + + // Serialize metadata from a caller-provided replica descriptor list. + // This is used when we need to persist an updated replica set *before* + // mutating local replicas (which may free/reuse memory). + std::string SerializeMetadataForOpLogFromReplicaDescriptors( + const UUID& client_id, uint64_t size, + const std::vector& replicas) const; + + // Pending durable mutations (etcd write retry queue) + // -------------------------------------------------- + // In HA mode, freeing/reusing MEMORY replicas before Standby observes the + // corresponding OpLog update can cause stale descriptors on Standby. + // If durable etcd write fails, we enqueue a pending mutation and retry + // asynchronously to avoid long-term memory retention. + enum class PendingMutationKind : uint8_t { + EVICT_MEM_REPLICAS = 1, // drop MEMORY replicas; persist PUT_END or REMOVE + CLEAR_ALL_REPLICAS = 2, // remove the whole key; persist REMOVE + CLEAR_REPLICAS_ON_SEGMENT = 3, // remove COMPLETE replicas on segment; persist PUT_END/REMOVE + }; + struct PendingMutation { + PendingMutationKind kind{PendingMutationKind::EVICT_MEM_REPLICAS}; + std::string key; + std::string segment_name; // only for CLEAR_REPLICAS_ON_SEGMENT + // OpLog entry to persist. If sequence_id==0, this is a deferred action and + // the worker will allocate a new OpLogEntry at execution time. + // If sequence_id>0, sequence_id is pre-allocated and MUST be persisted as-is + // (implements: "enqueue time seq_id fixed and smaller"). + OpLogEntry oplog_entry; + uint32_t attempt{0}; + std::chrono::steady_clock::time_point next_retry_at{}; + }; + + void EnqueuePendingMutation(PendingMutation m); + void PendingMutationWorker(); + bool ProcessPendingMutationOnce(PendingMutation& m); + + // Helper for etcd durable write (HA only): + // - Persist a pre-allocated OpLogEntry with small synchronous retries. + // - On failure, enqueue a PendingMutation (caller decides whether to proceed + // with local state changes; we do NOT block per-key). + ErrorCode PersistOpLogEntryWithSyncRetries(const OpLogEntry& entry) const; + void EnqueueRetryOnPersistFailure(const char* ctx, const OpLogEntry& entry, + ErrorCode persist_err, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // Higher-level helper that also handles: + // - STORE_USE_ETCD compile-time switch + // - enable_ha_ runtime switch + // + // Behavior: + // - HA + STORE_USE_ETCD: AllocateEntry -> Persist (sync retries) -> enqueue on failure + // - non-HA: Append to in-memory OpLog buffer only + // - HA but STORE_USE_ETCD disabled: no-op (best-effort; see constructor warning) + void AppendOrPersistOrEnqueue(const char* ctx, OpType type, + const std::string& key, + const std::string& payload, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // Lazy-payload variant: payload is computed only when needed. + // This is useful to avoid expensive metadata serialization when: + // - HA is enabled but STORE_USE_ETCD is disabled at compile time (no-op), or + // - the branch will not publish OpLog at all. + void AppendOrPersistOrEnqueueLazy( + const char* ctx, OpType type, const std::string& key, + const std::function& payload_factory, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // NOTE: + // We intentionally do NOT block subsequent operations for the same key when a + // durable OpLog write fails. Failed entries are retried asynchronously with the + // original pre-allocated sequence_id, and Standby handles gaps via timeout + + // late-arrival policy (apply late REMOVE/PUT_REVOKE, discard late PUT_END). + static constexpr size_t kNumShards = 1024; // Number of metadata shards // Sharded metadata maps and their mutexes @@ -684,6 +775,13 @@ class MasterService { std::shared_ptr allocation_strategy_; + // Pending durable mutation retry queue (HA only). + std::mutex pending_mutations_mutex_; + std::condition_variable pending_mutations_cv_; + std::deque pending_mutations_; + std::atomic pending_mutations_running_{false}; + std::thread pending_mutations_thread_; + // Discarded replicas management const std::chrono::seconds put_start_discard_timeout_sec_; const std::chrono::seconds put_start_release_timeout_sec_; diff --git a/mooncake-store/include/oplog_applier.h b/mooncake-store/include/oplog_applier.h index 2e611bbe26..cbab4623a8 100644 --- a/mooncake-store/include/oplog_applier.h +++ b/mooncake-store/include/oplog_applier.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -72,6 +73,21 @@ class OpLogApplier { */ size_t ProcessPendingEntries(); + // Promotion helper: + // Try to resolve current gaps ONCE (no waiting) by fetching missing/skipped + // sequence_ids from etcd. If an entry arrives late: + // - REMOVE / PUT_REVOKE: delete the key + // - PUT_END: discard + // + // This is used during Standby promotion so we don't block promotion on gaps, + // but still best-effort clean up potentially stale metadata. + struct GapResolveResult { + size_t attempted{0}; + size_t fetched{0}; + size_t applied_deletes{0}; + }; + GapResolveResult TryResolveGapsOnceForPromotion(size_t max_ids = 1024); + private: /** * @brief Check if the entry's sequence order is valid @@ -133,11 +149,19 @@ class OpLogApplier { // Track missing sequence IDs that we're waiting for std::map missing_sequence_ids_; + + // Sequence IDs we chose to skip (gap-timeout). If the late entry arrives: + // - REMOVE / PUT_REVOKE: delete the key (safe) + // - PUT_END: discard (do not resurrect potentially stale metadata) + std::map skipped_sequence_ids_; - uint64_t expected_sequence_id_{1}; + // Next expected global sequence_id. Read frequently from monitoring thread, + // updated by watch/apply thread. Use atomic to avoid data races. + std::atomic expected_sequence_id_{1}; // Constants for missing entry handling static constexpr int kMissingEntryWaitSeconds = 5; // Wait 5 seconds before requesting + static constexpr int kMissingEntrySkipSeconds = 3; // Wait 3 seconds then skip (avoid global stall) static constexpr int kMaxPendingEntries = 1000; // Max pending entries before giving up }; diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index b0f3dcdb1c..4abc6c8ae7 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -8,6 +8,10 @@ #include #include +#include + +#include "types.h" + namespace mooncake { // Forward declaration @@ -19,6 +23,8 @@ enum class OpType : uint8_t { PUT_END = 1, PUT_REVOKE = 2, REMOVE = 3, + // Deprecated: LEASE_RENEW is intentionally not recorded in OpLog in the + // current etcd-based hot-standby design (Standby relies on Primary DELETEs). LEASE_RENEW = 4, }; @@ -33,7 +39,9 @@ struct OpLogEntry { std::string payload; // Serialized extra data (optional) uint32_t checksum{0}; // Checksum of payload (implementation-defined) uint32_t prefix_hash{0}; // Hash of the entire key (for verification and optimization) - uint64_t key_sequence_id{0}; // Per-key sequence ID (for ordering guarantee) + // Deprecated: key_sequence_id is kept for backward compatibility only. + // Ordering is guaranteed by global sequence_id. + uint64_t key_sequence_id{0}; }; /** @@ -55,6 +63,34 @@ class OpLogManager { uint64_t Append(OpType type, const std::string& key, const std::string& payload = std::string()); + // Allocate a new OpLogEntry with a reserved sequence_id, append it to the + // in-memory buffer, and return the full entry. + // + // IMPORTANT: This will advance last_seq_id_ even if the caller later fails + // to persist it to etcd. This supports "seq pre-allocation" semantics where + // retries use the same (smaller) sequence_id. + OpLogEntry AllocateEntry(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Persist an already-allocated entry to etcd using its sequence_id. + // Does NOT modify sequence counters. + ErrorCode PersistEntryToEtcd(const OpLogEntry& entry) const; + + // Append a new entry and durably persist it to etcd (if EtcdOpLogStore is set). + // + // This is intended for operations that may free/reuse memory (e.g. REMOVE), + // where best-effort replication is unsafe: Standby must observe the DELETE + // before promotion, otherwise it may return stale descriptors that point to + // reused memory and cause silent data corruption. + // + // Design (updated for seq pre-allocation): + // - sequence_id is allocated first and never reused. + // - If etcd write fails, caller may retry PersistEntryToEtcd with the same + // entry (sequence_id fixed and "smaller" than later entries). + tl::expected AppendAndPersist( + OpType type, const std::string& key, + const std::string& payload = std::string()); + // Get the latest assigned sequence id. Returns 0 if no entry exists. uint64_t GetLastSequenceId() const; diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp index 2ad5ab1757..cc0775200c 100644 --- a/mooncake-store/src/etcd_oplog_store.cpp +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -19,6 +19,11 @@ EtcdOpLogStore::EtcdOpLogStore(const std::string& cluster_id, : cluster_id_(cluster_id), enable_latest_seq_batch_update_(enable_latest_seq_batch_update), last_update_time_(std::chrono::steady_clock::now()) { + // Normalize cluster_id to avoid accidental double slashes in etcd keys when + // caller passes a trailing '/' (master_view_key uses trailing '/', OpLog keys don't). + while (!cluster_id_.empty() && cluster_id_.back() == '/') { + cluster_id_.pop_back(); + } // Start batch update thread only for writers. if (enable_latest_seq_batch_update_) { batch_update_running_.store(true); diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index fb5912fa9e..c187b92733 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -286,6 +286,17 @@ std::unique_ptr HotStandbyService::Promote() { oplog_watcher_->Stop(); } + // Best-effort: resolve any outstanding gaps ONCE before promotion. + // Do NOT block promotion if gaps cannot be fetched. + if (oplog_applier_) { + auto res = oplog_applier_->TryResolveGapsOnceForPromotion(/*max_ids=*/1024); + if (res.attempted > 0) { + LOG(INFO) << "Promotion gap resolve (best-effort): attempted=" << res.attempted + << ", fetched=" << res.fetched + << ", applied_deletes=" << res.applied_deletes; + } + } + LOG(INFO) << "Final catch-up sync from etcd before promotion..."; EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); const size_t batch_size = 1000; diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 602e393d1e..937d275768 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -100,6 +100,39 @@ std::string MasterService::SerializeMetadataForOpLog(const ObjectMetadata& metad return json_str; } +std::string MasterService::SerializeMetadataForOpLogWithoutMemReplicas( + const ObjectMetadata& metadata) const { + MetadataPayload payload; + payload.client_id_first = metadata.client_id.first; + payload.client_id_second = metadata.client_id.second; + payload.size = metadata.size; + + payload.replicas.reserve(metadata.replicas.size()); + for (const auto& replica : metadata.replicas) { + if (replica.type() == ReplicaType::MEMORY) { + continue; + } + payload.replicas.push_back(replica.get_descriptor()); + } + + std::string json_str; + struct_json::to_json(payload, json_str); + return json_str; +} + +std::string MasterService::SerializeMetadataForOpLogFromReplicaDescriptors( + const UUID& client_id, uint64_t size, + const std::vector& replicas) const { + MetadataPayload payload; + payload.client_id_first = client_id.first; + payload.client_id_second = client_id.second; + payload.size = size; + payload.replicas = replicas; + std::string json_str; + struct_json::to_json(payload, json_str); + return json_str; +} + MasterService::MasterService(const MasterServiceConfig& config) : default_kv_lease_ttl_(config.default_kv_lease_ttl), default_kv_soft_pin_ttl_(config.default_kv_soft_pin_ttl), @@ -184,15 +217,51 @@ MasterService::MasterService(const MasterServiceConfig& config) "Recompile with -DSTORE_USE_ETCD=ON to enable etcd support."; } #endif + + // Start pending durable mutation retry thread (HA only). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + pending_mutations_running_.store(true); + pending_mutations_thread_ = + std::thread(&MasterService::PendingMutationWorker, this); + } +#endif } -// Helper function to append OpLog entry -// Note: In the new etcd-based design, OpLog will be written to etcd by EtcdOpLogStore -// This method only appends to OpLogManager's buffer for now +// Helper function to append an OpLog entry. +// In the current etcd-based design: +// - OpLogManager always appends to its in-memory buffer +// - If EtcdOpLogStore is configured (HA mode), OpLogManager also writes to etcd +// synchronously (best-effort; see OpLogManager::Append). void MasterService::AppendOpLogAndNotify(OpType type, const std::string& key, const std::string& payload) { oplog_manager_.Append(type, key, payload); - // TODO: In Phase 1, integrate with EtcdOpLogStore to write to etcd +} + +auto MasterService::AppendOpLogAndNotifyDurable(OpType type, const std::string& key, + const std::string& payload) + -> tl::expected { +#ifdef STORE_USE_ETCD + // In HA mode, EtcdOpLogStore should have been configured into OpLogManager. + // For safety, treat missing store as an error for durable ops. + // Best-effort synchronous retries to absorb transient etcd blips. + // + // IMPORTANT: + // sequence_id must be allocated ONCE (pre-allocation) and retried with the same + // OpLogEntry, otherwise multiple attempts would allocate multiple sequence_ids + // for a single logical operation. + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, payload); + ErrorCode err = PersistOpLogEntryWithSyncRetries(entry); + if (err == ErrorCode::OK) { + return entry.sequence_id; + } + return tl::make_unexpected(err); +#else + (void)type; + (void)key; + (void)payload; + return tl::make_unexpected(ErrorCode::ETCD_OPERATION_ERROR); +#endif } void MasterService::RestoreFromStandbySnapshot( @@ -277,6 +346,207 @@ MasterService::~MasterService() { if (client_monitor_thread_.joinable()) { client_monitor_thread_.join(); } + +#ifdef STORE_USE_ETCD + if (pending_mutations_running_.load()) { + pending_mutations_running_.store(false); + pending_mutations_cv_.notify_all(); + if (pending_mutations_thread_.joinable()) { + pending_mutations_thread_.join(); + } + } +#endif +} + +void MasterService::EnqueuePendingMutation(PendingMutation m) { + m.attempt = 0; + m.next_retry_at = std::chrono::steady_clock::now(); + { + std::lock_guard lg(pending_mutations_mutex_); + pending_mutations_.push_back(std::move(m)); + } + pending_mutations_cv_.notify_one(); +} + +ErrorCode MasterService::PersistOpLogEntryWithSyncRetries( + const OpLogEntry& entry) const { +#ifdef STORE_USE_ETCD + static constexpr int kSyncRetries = 3; + static constexpr int kBaseBackoffMs = 20; + ErrorCode persist_err = ErrorCode::ETCD_OPERATION_ERROR; + for (int attempt = 0; attempt < kSyncRetries; ++attempt) { + persist_err = oplog_manager_.PersistEntryToEtcd(entry); + if (persist_err == ErrorCode::OK) { + break; + } + std::this_thread::sleep_for( + std::chrono::milliseconds(kBaseBackoffMs * (1 << attempt))); + } + return persist_err; +#else + (void)entry; + return ErrorCode::ETCD_OPERATION_ERROR; +#endif +} + +void MasterService::EnqueueRetryOnPersistFailure( + const char* ctx, const OpLogEntry& entry, ErrorCode persist_err, + PendingMutationKind kind, const std::string& segment_name) { +#ifdef STORE_USE_ETCD + LOG(ERROR) << ctx << ": failed to persist OpLog to etcd, key=" + << entry.object_key << ", seq=" << entry.sequence_id + << ", err=" << persist_err << ". Enqueue retry."; + EnqueuePendingMutation(PendingMutation{ + kind, + entry.object_key, + segment_name, + /*oplog_entry=*/entry}); +#else + (void)ctx; + (void)entry; + (void)persist_err; + (void)kind; + (void)segment_name; +#endif +} + +void MasterService::AppendOrPersistOrEnqueue( + const char* ctx, OpType type, const std::string& key, + const std::string& payload, PendingMutationKind kind, + const std::string& segment_name) { +#ifdef STORE_USE_ETCD + if (enable_ha_) { + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, payload); + ErrorCode persist_err = PersistOpLogEntryWithSyncRetries(entry); + if (persist_err != ErrorCode::OK) { + EnqueueRetryOnPersistFailure(ctx, entry, persist_err, kind, segment_name); + } + } else { + AppendOpLogAndNotify(type, key, payload); + } +#else + // No etcd support at compile time: + // - non-HA: keep best-effort in-memory OpLog for debugging/consistency + // - HA: no-op (constructor already warns) + if (!enable_ha_) { + AppendOpLogAndNotify(type, key, payload); + } + (void)ctx; + (void)kind; + (void)segment_name; +#endif +} + +void MasterService::AppendOrPersistOrEnqueueLazy( + const char* ctx, OpType type, const std::string& key, + const std::function& payload_factory, + PendingMutationKind kind, const std::string& segment_name) { + std::string payload; + bool payload_ready = false; + auto get_payload = [&]() -> const std::string& { + if (!payload_ready) { + payload = payload_factory ? payload_factory() : std::string(); + payload_ready = true; + } + return payload; + }; + +#ifdef STORE_USE_ETCD + if (enable_ha_) { + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, get_payload()); + ErrorCode persist_err = PersistOpLogEntryWithSyncRetries(entry); + if (persist_err != ErrorCode::OK) { + EnqueueRetryOnPersistFailure(ctx, entry, persist_err, kind, segment_name); + } + } else { + AppendOpLogAndNotify(type, key, get_payload()); + } +#else + // No etcd support at compile time: + // - non-HA: keep best-effort in-memory OpLog for debugging/consistency + // - HA: no-op (constructor already warns) + if (!enable_ha_) { + AppendOpLogAndNotify(type, key, get_payload()); + } + (void)ctx; + (void)kind; + (void)segment_name; +#endif +} + +// Return true if processed successfully (done), false if should retry later. +bool MasterService::ProcessPendingMutationOnce(PendingMutation& m) { +#ifndef STORE_USE_ETCD + (void)m; + return true; +#else + const auto now = std::chrono::steady_clock::now(); + if (m.next_retry_at > now) { + return false; + } + + // Retrier responsibility: + // only persist the original pre-allocated OpLogEntry (fixed sequence_id) to etcd. + // Do NOT mutate local metadata here because the caller may have already moved on. + if (m.oplog_entry.sequence_id == 0) { + LOG(WARNING) << "PendingMutation has no pre-allocated OpLogEntry, drop. key=" + << m.key << ", kind=" << static_cast(m.kind); + return true; + } + + ErrorCode err = oplog_manager_.PersistEntryToEtcd(m.oplog_entry); + if (err != ErrorCode::OK) { + return false; + } + return true; +#endif +} + +void MasterService::PendingMutationWorker() { +#ifndef STORE_USE_ETCD + return; +#else + while (pending_mutations_running_.load()) { + PendingMutation m; + bool has_item = false; + { + std::unique_lock lk(pending_mutations_mutex_); + pending_mutations_cv_.wait_for(lk, std::chrono::milliseconds(200), [&] { + return !pending_mutations_running_.load() || !pending_mutations_.empty(); + }); + if (!pending_mutations_running_.load()) { + break; + } + if (pending_mutations_.empty()) { + continue; + } + m = std::move(pending_mutations_.front()); + pending_mutations_.pop_front(); + has_item = true; + } + if (!has_item) { + continue; + } + + const bool done = ProcessPendingMutationOnce(m); + if (done) { + continue; + } + + // Retry with exponential backoff (cap at 30s). + m.attempt++; + const uint32_t exp = std::min(m.attempt, 8); + const auto delay = std::chrono::milliseconds(200u * (1u << exp)); + const auto capped = std::min(delay, std::chrono::milliseconds(30000)); + m.next_retry_at = std::chrono::steady_clock::now() + capped; + + { + std::lock_guard lg(pending_mutations_mutex_); + pending_mutations_.push_back(std::move(m)); + } + pending_mutations_cv_.notify_one(); + } +#endif } auto MasterService::MountSegment(const Segment& segment, const UUID& client_id) @@ -365,10 +635,40 @@ void MasterService::ClearInvalidHandles() { MutexLocker lock(&shard.mutex); auto it = shard.metadata.begin(); while (it != shard.metadata.end()) { + // CleanupStaleHandles may remove MEMORY replicas whose allocator has + // become invalid (segment unmounted). If key remains valid (has disk + // replicas), Standby must receive an updated metadata payload that + // excludes those MEMORY replicas (Scheme A). if (CleanupStaleHandles(it->second)) { - // If the object is empty, we need to erase the iterator + // No replicas remain after cleanup -> key should be deleted. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("ClearInvalidHandles(REMOVE)", + OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, it->first); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, it->first); + } +#endif it = shard.metadata.erase(it); } else { + // Still has some replicas. If HA is enabled, publish updated + // metadata WITHOUT MEMORY replicas (safe superset update). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueueLazy( + "ClearInvalidHandles(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } +#endif ++it; } } @@ -583,6 +883,23 @@ auto MasterService::BatchReplicaClear( continue; } + // HA safety (Scheme A): + // This operation may free/reuse MEMORY replicas. Persist REMOVE to etcd + // BEFORE actually erasing local metadata. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("BatchReplicaClear(all)", OpType::REMOVE, + key, std::string(), + PendingMutationKind::CLEAR_ALL_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif + // Before erasing, decrement cache metrics for each COMPLETE replica for (const auto& replica : metadata.replicas) { if (replica.status() == ReplicaStatus::COMPLETE) { @@ -629,6 +946,45 @@ auto MasterService::BatchReplicaClear( continue; } + // HA safety (Scheme A): + // Removing replicas may free/reuse MEMORY replicas. Persist updated metadata + // BEFORE mutating metadata.replicas (which may free memory). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + // Build the remaining replica descriptor list after removal. + std::vector remove_mask(metadata.replicas.size(), false); + for (size_t idx : replicas_to_remove) { + if (idx < remove_mask.size()) { + remove_mask[idx] = true; + } + } + std::vector remaining; + remaining.reserve(metadata.replicas.size()); + for (size_t i = 0; i < metadata.replicas.size(); ++i) { + if (remove_mask[i]) { + continue; + } + remaining.emplace_back(metadata.replicas[i].get_descriptor()); + } + + if (remaining.empty()) { + AppendOrPersistOrEnqueue("BatchReplicaClear(partial REMOVE)", + OpType::REMOVE, key, std::string(), + PendingMutationKind::CLEAR_REPLICAS_ON_SEGMENT, + segment_name); + } else { + const std::string payload = + SerializeMetadataForOpLogFromReplicaDescriptors( + metadata.client_id, static_cast(metadata.size), + remaining); + AppendOrPersistOrEnqueue("BatchReplicaClear(partial PUT_END)", + OpType::PUT_END, key, payload, + PendingMutationKind::CLEAR_REPLICAS_ON_SEGMENT, + segment_name); + } + } +#endif + // Remove replicas on the specified segment (in reverse order to // maintain indices) for (auto it = replicas_to_remove.rbegin(); @@ -646,7 +1002,21 @@ auto MasterService::BatchReplicaClear( // If no valid replicas remain, erase the entire metadata if (metadata.replicas.empty() || !metadata.IsValid()) { +#ifndef STORE_USE_ETCD + // Non-HA: keep old behavior; HA already persisted REMOVE above. + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif accessor.Erase(); + } else { +#ifndef STORE_USE_ETCD + // Non-HA: best-effort update to keep future behavior consistent. + if (!enable_ha_) { + const std::string payload = SerializeMetadataForOpLog(metadata); + AppendOpLogAndNotify(OpType::PUT_END, key, payload); + } +#endif } cleared_keys.emplace_back(key); @@ -969,6 +1339,23 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, return tl::make_unexpected(ErrorCode::INVALID_WRITE); } + // HA behavior: + // Do NOT block subsequent ops for the same key if etcd write fails. + // We allocate sequence_id once and retry persisting this OpLogEntry + // asynchronously if needed. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("PutRevoke", OpType::PUT_REVOKE, key, std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::PUT_REVOKE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::PUT_REVOKE, key); + } +#endif + if (replica_type == ReplicaType::MEMORY) { MasterMetricManager::instance().dec_mem_cache_nums(); } else if (replica_type == ReplicaType::DISK) { @@ -986,9 +1373,6 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, accessor.Erase(); } - // Log the revoke operation so that standbys can roll back their metadata. - AppendOpLogAndNotify(OpType::PUT_REVOKE, key); - return {}; } @@ -1032,11 +1416,24 @@ auto MasterService::Remove(const std::string& key) return tl::make_unexpected(ErrorCode::REPLICA_IS_NOT_READY); } - // Remove object metadata - accessor.Erase(); + // HA behavior: + // If etcd write fails, enqueue retry but still proceed with local remove. + // Standby will handle gaps via timeout + late-arrival policy. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("Remove", OpType::REMOVE, key, std::string(), + PendingMutationKind::CLEAR_ALL_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif - // Log explicit remove so that standbys can delete the same key. - AppendOpLogAndNotify(OpType::REMOVE, key); + // Remove object metadata (may deallocate memory replicas) + accessor.Erase(); return {}; } @@ -1485,9 +1882,39 @@ void MasterService::BatchEvict(double evict_ratio_target, continue; } if (it->second.lease_timeout <= target_timeout) { - // Evict this object + // Evict this object (MEMORY replicas only). + // + // Scheme A: + // - If key remains valid after removing MEMORY replicas, + // durably persist a PUT_END carrying the updated metadata + // (without MEMORY replicas) before freeing memory. + // - If key becomes invalid (only had MEMORY replicas), + // durably persist REMOVE before freeing memory. total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { @@ -1549,9 +1976,32 @@ void MasterService::BatchEvict(double evict_ratio_target, !it->second.HasDiffRepStatus(ReplicaStatus::COMPLETE, ReplicaType::MEMORY) && it->second.HasMemReplica()) { - // Evict this object + // Evict this object (MEMORY replicas only). See Scheme A above. total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { @@ -1605,6 +2055,29 @@ void MasterService::BatchEvict(double evict_ratio_target, it->second.lease_timeout <= soft_target_timeout) { total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index 0029e6d6de..2a7e83f690 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -40,24 +40,63 @@ EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const { } bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { - // Check global sequence order (key_sequence_id is no longer used) - if (entry.sequence_id != expected_sequence_id_) { - // Global sequence violation - add to pending entries + // Global ordering only (key_sequence_id is deprecated and ignored). + // + // IMPORTANT: + // - Watch callbacks / retries may deliver duplicate or already-applied entries. + // - Those must be treated as no-op, not as "out-of-order pending", otherwise + // pending_entries_ can grow and the applier may appear stuck. + const uint64_t expected = expected_sequence_id_.load(); + if (entry.sequence_id < expected) { + // Late arrival of a previously-skipped gap entry: apply only if it's a delete/revoke. + bool was_skipped = false; + { + std::lock_guard lock(pending_mutex_); + auto it = skipped_sequence_ids_.find(entry.sequence_id); + if (it != skipped_sequence_ids_.end()) { + was_skipped = true; + skipped_sequence_ids_.erase(it); + } + } + if (was_skipped) { + if (entry.op_type == OpType::REMOVE || entry.op_type == OpType::PUT_REVOKE) { + // Safe: ensure we don't keep stale metadata. + if (entry.op_type == OpType::REMOVE) { + ApplyRemove(entry); + } else { + ApplyPutRevoke(entry); + } + return true; + } + // PUT_END (or others): discard to avoid resurrecting stale state. + VLOG(1) << "OpLogApplier: discard late skipped entry, op_type=" + << static_cast(entry.op_type) + << ", sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key; + return true; + } + + VLOG(2) << "OpLogApplier: skip already-applied entry, sequence_id=" + << entry.sequence_id << ", expected=" << expected + << ", key=" << entry.object_key; + return true; // consumed (no-op) + } + if (entry.sequence_id > expected) { + // Future entry - store into pending, wait for the gap to be filled. std::lock_guard lock(pending_mutex_); - - // Check if we've exceeded max pending entries + if (pending_entries_.size() >= static_cast(kMaxPendingEntries)) { LOG(ERROR) << "OpLogApplier: too many pending entries (" << pending_entries_.size() << "), discarding entry sequence_id=" << entry.sequence_id << ", key=" << entry.object_key; return false; } - + pending_entries_[entry.sequence_id] = entry; - VLOG(1) << "OpLogApplier: sequence order violation, sequence_id=" - << entry.sequence_id << ", expected=" << expected_sequence_id_ + VLOG(1) << "OpLogApplier: future entry buffered, sequence_id=" + << entry.sequence_id << ", expected=" << expected << ", key=" << entry.object_key - << ", added to pending entries (total: " << pending_entries_.size() << ")"; + << ", pending_entries=" << pending_entries_.size(); return false; } @@ -81,7 +120,7 @@ bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { } // Update expected sequence ID - expected_sequence_id_ = entry.sequence_id + 1; + expected_sequence_id_.store(entry.sequence_id + 1); // Try to process pending entries ProcessPendingEntries(); @@ -107,43 +146,58 @@ uint64_t OpLogApplier::GetKeySequenceId(const std::string& key) const { } uint64_t OpLogApplier::GetExpectedSequenceId() const { - return expected_sequence_id_; + return expected_sequence_id_.load(); } void OpLogApplier::Recover(uint64_t last_applied_sequence_id) { - expected_sequence_id_ = last_applied_sequence_id + 1; + expected_sequence_id_.store(last_applied_sequence_id + 1); LOG(INFO) << "OpLogApplier: recovered from sequence_id=" << last_applied_sequence_id - << ", expected_sequence_id set to=" << expected_sequence_id_; + << ", expected_sequence_id set to=" << expected_sequence_id_.load(); } size_t OpLogApplier::ProcessPendingEntries() { - // Check for missing sequence IDs and request them if needed (before acquiring lock) + // Check for missing sequence IDs, possibly skip after timeout, and/or request them. uint64_t missing_seq_to_request = 0; + uint64_t skipped_count = 0; { std::lock_guard lock(pending_mutex_); - if (!pending_entries_.empty()) { - uint64_t first_pending_seq = pending_entries_.begin()->first; - if (first_pending_seq > expected_sequence_id_) { - // There's a gap - we're missing entries between expected_sequence_id_ and first_pending_seq - uint64_t missing_seq = expected_sequence_id_; - auto missing_it = missing_sequence_ids_.find(missing_seq); - auto now = std::chrono::steady_clock::now(); - - if (missing_it == missing_sequence_ids_.end()) { - // First time we see this missing sequence - record the time - missing_sequence_ids_[missing_seq] = now; - ScheduleWaitForMissingEntries(missing_seq); - } else { - // Check if we've waited long enough - auto wait_duration = std::chrono::duration_cast( - now - missing_it->second); - if (wait_duration.count() >= kMissingEntryWaitSeconds) { - // Mark for requesting (we'll do it after releasing the lock) - missing_seq_to_request = missing_seq; - } - } + auto now = std::chrono::steady_clock::now(); + for (;;) { + if (pending_entries_.empty()) { + break; + } + const uint64_t first_pending_seq = pending_entries_.begin()->first; + const uint64_t expected = expected_sequence_id_.load(); + if (first_pending_seq <= expected) { + break; + } + + // There's a gap: expected is missing. + const uint64_t missing_seq = expected; + auto it = missing_sequence_ids_.find(missing_seq); + if (it == missing_sequence_ids_.end()) { + missing_sequence_ids_[missing_seq] = now; + ScheduleWaitForMissingEntries(missing_seq); + break; } + + const auto waited = std::chrono::duration_cast(now - it->second); + + // Skip after 3s to avoid global stall (user requested behavior). + if (waited.count() >= kMissingEntrySkipSeconds) { + skipped_sequence_ids_[missing_seq] = now; + missing_sequence_ids_.erase(missing_seq); + expected_sequence_id_.store(missing_seq + 1); + skipped_count++; + continue; // may skip multiple consecutive gaps + } + + // Optionally request from etcd after a longer wait (best-effort). + if (waited.count() >= kMissingEntryWaitSeconds) { + missing_seq_to_request = missing_seq; + } + break; } } @@ -169,7 +223,8 @@ size_t OpLogApplier::ProcessPendingEntries() { } auto it = pending_entries_.begin(); - if (it->first != expected_sequence_id_) { + const uint64_t expected = expected_sequence_id_.load(); + if (it->first != expected) { break; // still waiting for earlier sequence_id } @@ -198,7 +253,7 @@ size_t OpLogApplier::ProcessPendingEntries() { break; } - expected_sequence_id_ = entry_copy.sequence_id + 1; + expected_sequence_id_.store(entry_copy.sequence_id + 1); { std::lock_guard lock(pending_mutex_); @@ -222,25 +277,105 @@ size_t OpLogApplier::ProcessPendingEntries() { ++it; } } + + // Clean up old skipped sequence IDs too (avoid unbounded growth). + for (auto it = skipped_sequence_ids_.begin(); it != skipped_sequence_ids_.end();) { + auto age = std::chrono::duration_cast(now - it->second); + if (age.count() > 60) { + it = skipped_sequence_ids_.erase(it); + } else { + ++it; + } + } + } + + if (skipped_count > 0) { + LOG(WARNING) << "OpLogApplier: skipped " << skipped_count + << " missing sequence_id(s) after timeout, expected_sequence_id now=" + << expected_sequence_id_.load(); } if (processed_count > 0) { LOG(INFO) << "OpLogApplier: processed " << processed_count << " pending entries, expected_sequence_id now=" - << expected_sequence_id_; + << expected_sequence_id_.load(); } return processed_count; } +OpLogApplier::GapResolveResult OpLogApplier::TryResolveGapsOnceForPromotion( + size_t max_ids) { + GapResolveResult r; +#ifdef STORE_USE_ETCD + EtcdOpLogStore* store = GetEtcdOpLogStore(); + if (store == nullptr) { + return r; + } + + std::vector gap_ids; + gap_ids.reserve(max_ids); + { + std::lock_guard lock(pending_mutex_); + for (const auto& kv : missing_sequence_ids_) { + if (gap_ids.size() >= max_ids) break; + gap_ids.push_back(kv.first); + } + for (const auto& kv : skipped_sequence_ids_) { + if (gap_ids.size() >= max_ids) break; + gap_ids.push_back(kv.first); + } + } + + if (gap_ids.empty()) { + return r; + } + + std::sort(gap_ids.begin(), gap_ids.end()); + gap_ids.erase(std::unique(gap_ids.begin(), gap_ids.end()), gap_ids.end()); + + r.attempted = gap_ids.size(); + for (uint64_t seq : gap_ids) { + OpLogEntry e; + ErrorCode err = store->ReadOpLog(seq, e); + if (err != ErrorCode::OK) { + continue; + } + r.fetched++; + + // Apply policy: only delete/revoke; drop PUT_END. + if (e.op_type == OpType::REMOVE) { + ApplyRemove(e); + r.applied_deletes++; + } else if (e.op_type == OpType::PUT_REVOKE) { + ApplyPutRevoke(e); + r.applied_deletes++; + } + } + + // Clear gaps we attempted so promotion won't keep retrying them. + { + std::lock_guard lock(pending_mutex_); + for (uint64_t seq : gap_ids) { + missing_sequence_ids_.erase(seq); + skipped_sequence_ids_.erase(seq); + } + } + return r; +#else + (void)max_ids; + return r; +#endif +} + bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { // Only check global sequence order. // key_sequence_id is no longer used for ordering. - return entry.sequence_id == expected_sequence_id_; + return entry.sequence_id == expected_sequence_id_.load(); } void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) { - // Payload contains serialized metadata (replicas, size, lease) in JSON format. + // Payload contains serialized metadata (replicas, size, etc.) in JSON format. // Deserialize the payload immediately and store structured metadata. // This allows Standby to serve requests immediately after promotion. diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index 3734b3532d..a9f5faa5bf 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -60,6 +60,49 @@ uint64_t OpLogManager::Append(OpType type, const std::string& key, return last_seq_id_; } +OpLogEntry OpLogManager::AllocateEntry(OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry entry; + entry.op_type = type; + entry.object_key = key; + entry.payload = payload; + entry.timestamp_ms = NowMs(); + entry.checksum = ComputeChecksum(entry.payload); + entry.prefix_hash = ComputePrefixHash(entry.object_key); + + std::unique_lock lock(mutex_); + entry.sequence_id = ++last_seq_id_; + entry.key_sequence_id = entry.sequence_id; // deprecated + + if (buffer_.size() >= kMaxBufferEntries_) { + buffer_.pop_front(); + ++first_seq_id_; + } + buffer_.emplace_back(entry); + return entry; +} + +ErrorCode OpLogManager::PersistEntryToEtcd(const OpLogEntry& entry) const { + std::shared_lock lock(mutex_); + auto store = etcd_oplog_store_; + lock.unlock(); + if (!store) { + return ErrorCode::ETCD_OPERATION_ERROR; + } + return store->WriteOpLog(entry); +} + +tl::expected OpLogManager::AppendAndPersist( + OpType type, const std::string& key, const std::string& payload) { + // Seq pre-allocation semantics: allocate first, then persist. + OpLogEntry entry = AllocateEntry(type, key, payload); + ErrorCode err = PersistEntryToEtcd(entry); + if (err != ErrorCode::OK) { + return tl::make_unexpected(err); + } + return entry.sequence_id; +} + uint64_t OpLogManager::GetLastSequenceId() const { std::shared_lock lock(mutex_); return last_seq_id_; diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp index 634fd5959b..3fa15fdcb4 100644 --- a/mooncake-store/src/oplog_watcher.cpp +++ b/mooncake-store/src/oplog_watcher.cpp @@ -25,6 +25,10 @@ OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, if (applier_ == nullptr) { LOG(FATAL) << "OpLogApplier cannot be null"; } + // Normalize cluster_id to avoid double slashes in watch prefix. + while (!cluster_id_.empty() && cluster_id_.back() == '/') { + cluster_id_.pop_back(); + } } OpLogWatcher::~OpLogWatcher() { @@ -227,6 +231,11 @@ void OpLogWatcher::WatchOpLog() { // The watch is now running in the background (via Go goroutine) // We just need to keep the thread alive until Stop() is called or watch fails while (running_.load() && watch_healthy_.load()) { + // Drive pending/missing handling even when no new watch events arrive. + // Without this, a single out-of-order arrival could park entries in + // pending_entries_ forever if the missing entry isn't delivered via watch + // (but exists in etcd and could be fetched). + (void)applier_->ProcessPendingEntries(); std::this_thread::sleep_for(std::chrono::milliseconds(100)); // Periodically check watch health @@ -374,7 +383,14 @@ void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& v // Apply the OpLog entry if (applier_->ApplyOpLogEntry(entry)) { - last_processed_sequence_id_.store(entry.sequence_id); + // last_processed_sequence_id_ must be monotonic. We may "consume" duplicate + // / already-applied entries (entry.sequence_id < expected) as no-ops, so + // never regress this counter. + uint64_t cur = last_processed_sequence_id_.load(); + while (entry.sequence_id > cur && + !last_processed_sequence_id_.compare_exchange_weak(cur, entry.sequence_id)) { + // retry + } consecutive_errors_.store(0); // Reset error counter on success reconnect_count_.store(0); // Reset reconnect counter on success VLOG(2) << "Applied OpLog entry: sequence_id=" << entry.sequence_id From a55525c76306cadcac509e4a2965f787ce63ed92 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 5 Jan 2026 09:33:18 +0800 Subject: [PATCH 38/59] fix dependency --- dependencies.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dependencies.sh b/dependencies.sh index 80f1c3ba69..ebcafe4221 100755 --- a/dependencies.sh +++ b/dependencies.sh @@ -121,6 +121,7 @@ SYSTEM_PACKAGES="build-essential \ libcurl4-openssl-dev \ libhiredis-dev \ libjemalloc-dev \ + libxxhash-dev \ pkg-config \ patchelf" From 44449f7f7d39e061dc6f1a349067dee4b3203fd6 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 5 Jan 2026 09:36:44 +0800 Subject: [PATCH 39/59] fix --- mooncake-store/include/oplog_manager.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index 4abc6c8ae7..3d9a0504b5 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -24,7 +24,7 @@ enum class OpType : uint8_t { PUT_REVOKE = 2, REMOVE = 3, // Deprecated: LEASE_RENEW is intentionally not recorded in OpLog in the - // current etcd-based hot-standby design (Standby relies on Primary DELETEs). + // current etcd-based hot-standby design (Standby relies on Primary DELETE operations). LEASE_RENEW = 4, }; From c7fcb8bb88b09c99a06df999d81c9b049ea568f1 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 5 Jan 2026 09:40:12 +0800 Subject: [PATCH 40/59] fix ci --- mooncake-store/src/rpc_service.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index 261ffa330f..df9fdb9938 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -33,9 +33,11 @@ WrappedMasterService::WrappedMasterService( init_http_server(); // ReplicationService removed - using etcd-based OpLog sync instead - // TODO: In Phase 1, initialize EtcdOpLogStore and integrate with OpLogManager + // TODO: In Phase 1, initialize EtcdOpLogStore and integrate with + // OpLogManager if (config.enable_ha) { - LOG(INFO) << "HA mode enabled - etcd-based OpLog sync will be implemented in Phase 1"; + LOG(INFO) << "HA mode enabled - etcd-based OpLog sync will be " + "implemented in Phase 1"; } if (config.enable_metric_reporting) { @@ -56,16 +58,17 @@ WrappedMasterService::~WrappedMasterService() { if (metric_report_thread_.joinable()) { metric_report_thread_.join(); } - + // ReplicationService removed - using etcd-based OpLog sync instead - + http_server_.stop(); } void WrappedMasterService::RestoreFromStandby( const std::vector>& snapshot, uint64_t initial_oplog_sequence_id) { - master_service_.RestoreFromStandbySnapshot(snapshot, initial_oplog_sequence_id); + master_service_.RestoreFromStandbySnapshot(snapshot, + initial_oplog_sequence_id); } void WrappedMasterService::init_http_server() { From 4ac8031905272f476751a810c69d794d5a48e2bd Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 5 Jan 2026 09:55:34 +0800 Subject: [PATCH 41/59] delete doc --- doc/en/diagrams/oplog-data-flow.puml | 58 -- doc/en/diagrams/oplog-failover-sequence.puml | 66 -- .../oplog-hot-standby-architecture.puml | 65 -- doc/en/rfc-oplog-hot-standby-complete.md | 318 -------- .../etcd-hot-standby-architecture.puml | 139 ---- .../etcd-hot-standby-diagrams-README.md | 127 --- doc/zh/diagrams/etcd-hot-standby-flow.puml | 173 ---- .../diagrams/etcd-hot-standby-sequence.puml | 259 ------ doc/zh/diagrams/mooncake-transfer-flow.puml | 80 -- doc/zh/diagrams/oplog-data-flow.puml | 58 -- doc/zh/diagrams/oplog-failover-sequence.puml | 66 -- .../oplog-hot-standby-architecture.puml | 65 -- ...log-hot-standby-complete-architecture.puml | 123 --- doc/zh/rfc-batched-delete-events-via-etcd.md | 455 ----------- doc/zh/rfc-batched-delete-timing-issues.md | 419 ---------- doc/zh/rfc-delete-via-etcd-solution.md | 427 ---------- .../rfc-dragonflydb-as-consistency-store.md | 313 -------- doc/zh/rfc-oplog-cleanup-start-sequence-id.md | 507 ------------ doc/zh/rfc-oplog-hot-standby-complete.md | 364 --------- doc/zh/rfc-oplog-hot-standby-promotion.md | 41 - doc/zh/rfc-oplog-implementation-plan.md | 728 ----------------- doc/zh/rfc-oplog-key-sequence-map-cleanup.md | 253 ------ ...g-rollback-replay-on-sequence-violation.md | 653 ---------------- doc/zh/rfc-oplog-via-etcd-complete-design.md | 738 ------------------ doc/zh/rfc-standby-no-response-handling.md | 355 --------- ...-standby-promotion-lease-initialization.md | 439 ----------- doc/zh/rfc-standby-service-integration.md | 673 ---------------- 27 files changed, 7962 deletions(-) delete mode 100644 doc/en/diagrams/oplog-data-flow.puml delete mode 100644 doc/en/diagrams/oplog-failover-sequence.puml delete mode 100644 doc/en/diagrams/oplog-hot-standby-architecture.puml delete mode 100644 doc/en/rfc-oplog-hot-standby-complete.md delete mode 100644 doc/zh/diagrams/etcd-hot-standby-architecture.puml delete mode 100644 doc/zh/diagrams/etcd-hot-standby-diagrams-README.md delete mode 100644 doc/zh/diagrams/etcd-hot-standby-flow.puml delete mode 100644 doc/zh/diagrams/etcd-hot-standby-sequence.puml delete mode 100644 doc/zh/diagrams/mooncake-transfer-flow.puml delete mode 100644 doc/zh/diagrams/oplog-data-flow.puml delete mode 100644 doc/zh/diagrams/oplog-failover-sequence.puml delete mode 100644 doc/zh/diagrams/oplog-hot-standby-architecture.puml delete mode 100644 doc/zh/diagrams/oplog-hot-standby-complete-architecture.puml delete mode 100644 doc/zh/rfc-batched-delete-events-via-etcd.md delete mode 100644 doc/zh/rfc-batched-delete-timing-issues.md delete mode 100644 doc/zh/rfc-delete-via-etcd-solution.md delete mode 100644 doc/zh/rfc-dragonflydb-as-consistency-store.md delete mode 100644 doc/zh/rfc-oplog-cleanup-start-sequence-id.md delete mode 100644 doc/zh/rfc-oplog-hot-standby-complete.md delete mode 100644 doc/zh/rfc-oplog-hot-standby-promotion.md delete mode 100644 doc/zh/rfc-oplog-implementation-plan.md delete mode 100644 doc/zh/rfc-oplog-key-sequence-map-cleanup.md delete mode 100644 doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md delete mode 100644 doc/zh/rfc-oplog-via-etcd-complete-design.md delete mode 100644 doc/zh/rfc-standby-no-response-handling.md delete mode 100644 doc/zh/rfc-standby-promotion-lease-initialization.md delete mode 100644 doc/zh/rfc-standby-service-integration.md diff --git a/doc/en/diagrams/oplog-data-flow.puml b/doc/en/diagrams/oplog-data-flow.puml deleted file mode 100644 index 32682eda9e..0000000000 --- a/doc/en/diagrams/oplog-data-flow.puml +++ /dev/null @@ -1,58 +0,0 @@ -@startuml oplog-data-flow -!theme plain -skinparam backgroundColor #FFFFFF -skinparam sequenceMessageAlign center - -actor Client -participant "Primary Master" as Primary -participant "OpLogManager" as OplogMgr -participant "EtcdOpLogStore" as EtcdStore -database etcd -participant "OpLogWatcher" as Watcher -participant "OpLogApplier" as Applier -participant "Standby Master" as Standby - -== PUT_END Operation Flow == - -Client -> Primary: PutEnd(key, ...) -activate Primary -Primary -> OplogMgr: Append(PUT_END, key) -activate OplogMgr -OplogMgr -> OplogMgr: Generate sequence_id\nGenerate key_sequence_id -OplogMgr -> EtcdStore: WriteOpLog(entry) -activate EtcdStore -EtcdStore -> etcd: PUT /oplog/{seq} -activate etcd -etcd --> EtcdStore: Success -deactivate etcd -EtcdStore --> OplogMgr: Success -deactivate EtcdStore -OplogMgr --> Primary: sequence_id -deactivate OplogMgr -Primary --> Client: Success -deactivate Primary - -== Standby Synchronization Flow == - -etcd -> Watcher: Watch Event (New OpLog) -activate Watcher -Watcher -> Applier: ApplyOpLogEntry(entry) -activate Applier -Applier -> Applier: CheckSequenceOrder() -alt Order Correct - Applier -> Standby: UpdateMetadata(key, ...) - activate Standby - Standby --> Applier: Success - deactivate Standby -else Order Violation - Applier -> Applier: RollbackAndReplay() - Applier -> etcd: ReadOpLogForKey() - etcd --> Applier: OpLog Entries - Applier -> Applier: Replay OpLog -end -Applier --> Watcher: Success -deactivate Applier -deactivate Watcher - -@enduml - diff --git a/doc/en/diagrams/oplog-failover-sequence.puml b/doc/en/diagrams/oplog-failover-sequence.puml deleted file mode 100644 index f6da4cc1f8..0000000000 --- a/doc/en/diagrams/oplog-failover-sequence.puml +++ /dev/null @@ -1,66 +0,0 @@ -@startuml oplog-failover-sequence -!theme plain -skinparam backgroundColor #FFFFFF -skinparam sequenceMessageAlign center - -participant "Primary Master" as Primary -database etcd -participant "Standby Master" as Standby -participant "MasterServiceSupervisor" as Supervisor -participant "HotStandbyService" as HotStandby -participant "OpLogWatcher" as Watcher -participant "OpLogApplier" as Applier - -== Normal Operation Phase == - -Primary -> etcd: KeepAlive Lease -etcd -> Supervisor: Watch Leader (Exists) -activate Supervisor -Supervisor -> HotStandby: StartStandby() -activate HotStandby -HotStandby -> Watcher: Start() -activate Watcher -Watcher -> etcd: Watch OpLog -etcd -> Watcher: OpLog Events -Watcher -> Applier: ApplyOpLogEntry() -activate Applier -Applier -> Standby: UpdateMetadata() -deactivate Applier -deactivate Watcher -deactivate HotStandby -deactivate Supervisor - -== Primary Failure == - -Primary -x etcd: Lease Expired (Failure) -etcd -> Supervisor: Leader Deleted Event -activate Supervisor -Supervisor -> HotStandby: Stop() -activate HotStandby -HotStandby -> Watcher: Stop() -deactivate Watcher -deactivate HotStandby - -== Standby Promotion to Primary == - -Supervisor -> Standby: Promote() -activate Standby -Standby -> Standby: Initialize Lease\nClean Expired metadata -Standby -> etcd: ElectLeader() -activate etcd -etcd -> Standby: Leader Elected -deactivate etcd -Standby -> Supervisor: Primary Mode -deactivate Standby -deactivate Supervisor - -note over Standby - 1. Stop Standby service - 2. Iterate all metadata - 3. Grant default lease to objects with lease=0 - 4. Perform complete metadata cleanup - 5. Start leader election -end note - -@enduml - diff --git a/doc/en/diagrams/oplog-hot-standby-architecture.puml b/doc/en/diagrams/oplog-hot-standby-architecture.puml deleted file mode 100644 index 6aa7e94f28..0000000000 --- a/doc/en/diagrams/oplog-hot-standby-architecture.puml +++ /dev/null @@ -1,65 +0,0 @@ -@startuml oplog-hot-standby-architecture -!theme plain -skinparam backgroundColor #FFFFFF -skinparam componentStyle rectangle -skinparam defaultFontSize 12 - -package "Primary Master" #E8F4F8 { - component [MasterService] as MasterService - component [OpLogManager] as OpLogManager - component [EtcdOpLogStore] as EtcdOpLogStore - - MasterService --> OpLogManager : Record Operations - OpLogManager --> EtcdOpLogStore : Write OpLog -} - -package "etcd Cluster" #FFF4E6 { - database [etcd] as etcd -} - -package "Standby Master" #F0F8E8 { - component [MasterServiceSupervisor] as Supervisor - component [HotStandbyService] as HotStandby - component [OpLogWatcher] as Watcher - component [OpLogApplier] as Applier - component [MetadataStore] as MetadataStore - - Supervisor --> HotStandby : Start/Stop - HotStandby --> Watcher : Watch OpLog - Watcher --> Applier : Apply OpLog - Applier --> MetadataStore : Update metadata -} - -EtcdOpLogStore --> etcd : Write OpLog -etcd --> Watcher : Watch Events - -note right of OpLogManager - **Responsibilities**: - - Generate sequence_id - - Generate key_sequence_id - - Maintain memory buffer -end note - -note right of Applier - **Responsibilities**: - - Check order - - Handle out-of-order - - Clean expired entries -end note - -note right of MasterService - **Operations**: - - PutEnd() - - Remove() - - Eviction() -end note - -note right of etcd - **Key Design**: - - /oplog/{cluster_id}/{sequence_id} - - /oplog/{cluster_id}/latest - - Watch API -end note - -@enduml - diff --git a/doc/en/rfc-oplog-hot-standby-complete.md b/doc/en/rfc-oplog-hot-standby-complete.md deleted file mode 100644 index 77e6684ff1..0000000000 --- a/doc/en/rfc-oplog-hot-standby-complete.md +++ /dev/null @@ -1,318 +0,0 @@ -# OpLog Hot-Standby Synchronization based on etcd - Complete RFC - -## 1. Background - -### 1.1 Current System Architecture - -Mooncake Store is a high-performance distributed KV cache storage engine designed specifically for LLM inference scenarios. The system adopts a Master-Client architecture: - -- **Master Service**: Manages object metadata, space allocation, node management, etc. -- **Client**: Acts as a storage server providing memory segments while also serving as a client to handle application requests - -### 1.2 High Availability Requirements - -The current system supports two deployment modes: - -1. **Default Mode**: Single Master node, simple deployment but with single point of failure risk -2. **High Availability Mode (unstable)**: Multiple Master nodes coordinated through etcd for leader election - -**Issues**: - -- While HA mode implements leader election, Standby Masters do not perform any operations during the waiting period -- No data synchronization mechanism is implemented; metadata may be incomplete when Standby is promoted to Primary -- Lack of reliable primary-standby data synchronization solution - -### 1.3 Business Scenarios - -In LLM inference scenarios, Master Service requires: -- **High Availability**: Fast failover when Master fails, minimizing service interruption time -- **Data Consistency**: Standby must maintain data consistency with Primary -- **Fast Recovery**: Quick service recovery after failure without lengthy data reconstruction - -### 1.4 Problems with Current Solution - -1. **No Data Synchronization**: Standby Master does not perform any data synchronization operations during the election waiting period -2. **Metadata Loss Risk**: After Primary failure, metadata may be incomplete when Standby is promoted -3. **Long Recovery Time**: Need to re-collect metadata from Client nodes, resulting in long recovery time -4. **Data Inconsistency**: Cannot guarantee data consistency between Standby and Primary - -## 2. Goals - -### 2.1 Primary Goals - -1. **Implement Reliable Primary-Standby Data Synchronization** - - Synchronize all metadata change operations from Primary Master to Standby Master - - Guarantee data consistency between Standby and Primary - -2. **Fast Failure Recovery** - - Standby can quickly promote to Primary after Primary failure - - Complete metadata when promoted, no lengthy reconstruction required - -3. **Minimize OpLog Size** - - Only record critical state change operations (PUT, DELETE) - - Do not record high-frequency but non-critical operations like lease renewals - -4. **Integration with Existing System** - - Integrate with existing snapshot mechanism - - Integrate with existing leader election mechanism - - Do not affect normal operation of existing features - -### 2.2 Non-Functional Goals - -1. **Performance**: OpLog synchronization should not significantly impact Primary performance -2. **Reliability**: Leverage etcd's strong consistency to guarantee data reliability -3. **Scalability**: Support multiple Standby Masters -4. **Maintainability**: Simple implementation, easy to understand and maintain - -## 3. Proposal - -### 3.1 Core Design Approach - -**Use etcd as an intermediate reliability component to implement OpLog primary-standby synchronization**: - -1. **OpLog Mechanism**: Primary Master records all state change operations to OpLog -2. **etcd Storage**: OpLog is written to etcd, leveraging etcd's strong consistency and persistence capabilities -3. **Watch Mechanism**: Standby Master receives OpLog in real-time through etcd Watch mechanism -4. **Ordering Guarantee**: Guarantee operation order through global sequence_id and key-level key_sequence_id - -### 3.2 Architecture Design - -#### 3.2.1 Overall Architecture - -The overall architecture diagram shows the interaction relationships between Primary Master, etcd Cluster, and Standby Master: - -![OpLog Hot-Standby Architecture](./diagrams/oplog-hot-standby-architecture.puml) - -**Architecture Description**: -- **Primary Master**: Handles client requests, records OpLog and writes to etcd -- **etcd Cluster**: Acts as intermediate storage, providing strong consistency and Watch mechanism -- **Standby Master**: Receives OpLog in real-time by watching etcd and applies to local metadata store - -#### 3.2.2 Data Flow Diagram - -The data flow diagram shows the complete flow from Client request to Standby synchronization: - -![OpLog Data Flow](./diagrams/oplog-data-flow.puml) - -**Flow Description**: -1. Client sends `PutEnd` request to Primary Master -2. Primary Master records operation through `OpLogManager`, generating sequence_id -3. `EtcdOpLogStore` writes OpLog to etcd -4. etcd notifies Standby Master through Watch mechanism -5. `OpLogWatcher` receives events and passes to `OpLogApplier` -6. `OpLogApplier` checks order and applies to Standby's metadata store - -#### 3.2.3 Failover Sequence - -The failover sequence diagram shows the complete process from Primary failure to Standby promotion to Primary: - -![OpLog Failover Sequence](./diagrams/oplog-failover-sequence.puml) - -**Flow Description**: -1. **Normal Operation**: Primary maintains Lease, Standby continuously synchronizes OpLog through Watch -2. **Primary Failure**: Primary's Lease expires, etcd notifies Standby -3. **Standby Promotion**: Stop Standby service, initialize Lease, clean expired metadata, start leader election - -### 3.3 Core Component Design - -#### 3.3.1 OpLogManager (Primary Side) - -**Responsibilities**: -- Record all state change operations (PUT_END, PUT_REVOKE, REMOVE) -- Generate global sequence_id and key-level key_sequence_id -- Maintain memory buffer (for fast queries) - -**Key Methods**: -```cpp -class OpLogManager { - uint64_t Append(OpType type, const std::string& key, - const std::string& payload = ""); - std::vector GetEntriesSince(uint64_t since_seq_id, - size_t limit = 1000) const; - uint64_t GetLastSequenceId() const; -}; -``` - -#### 3.3.2 EtcdOpLogStore (Primary Side) - -**Responsibilities**: -- Write OpLog to etcd -- Update latest sequence_id -- Record snapshot corresponding sequence_id -- Clean up old OpLog - -**etcd Key Design**: -- OpLog Entry: `mooncake-store/oplog/{cluster_id}/{sequence_id}` -- Latest Sequence ID: `mooncake-store/oplog/{cluster_id}/latest` -- Snapshot Sequence ID: `mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` - -#### 3.3.3 OpLogWatcher (Standby Side) - -**Responsibilities**: -- Watch etcd OpLog changes -- Read historical OpLog (for initial synchronization) -- Process Watch events and pass to OpLogApplier - -**Key Methods**: -```cpp -class OpLogWatcher { - void Start(); - void Stop(); - bool ReadOpLogSince(uint64_t start_seq_id, - std::vector& entries); -}; -``` - -#### 3.3.4 OpLogApplier (Standby Side) - -**Responsibilities**: -- Apply OpLog Entry to local metadata store -- Check global and key-level order -- Handle sequence number discontinuities and out-of-order cases -- Periodically clean up key_sequence_map_ (memory optimization) - -**Key Methods**: -```cpp -class OpLogApplier { - bool ApplyOpLogEntry(const OpLogEntry& entry); - bool CheckSequenceOrder(const OpLogEntry& entry); - void CleanupStaleKeySequences(); -}; -``` - -#### 3.3.5 HotStandbyService (Standby Side) - -**Responsibilities**: -- Manage Standby mode lifecycle -- Coordinate OpLogWatcher and OpLogApplier -- Handle Standby promotion to Primary logic - -**Key Methods**: -```cpp -class HotStandbyService { - void StartStandby(); - void Stop(); - void Promote(); -}; -``` - -### 3.4 OpLog Entry Data Structure - -```cpp -struct OpLogEntry { - uint64_t sequence_id{0}; // Globally monotonically increasing sequence - uint64_t timestamp_ms{0}; // Timestamp (milliseconds) - OpType op_type{OpType::PUT_END}; // PUT_END, PUT_REVOKE, REMOVE - std::string object_key; // Object key - std::string payload; // Optional payload (carries replica info for PUT_END) - uint32_t checksum{0}; // Checksum - uint32_t prefix_hash{0}; // Key prefix hash - uint64_t key_sequence_id{0}; // Per-key operation sequence (for ordering guarantee) -}; -``` - -**JSON Serialization Format**: -```json -{ - "sequence_id": 12345, - "timestamp": 1704110400123, - "op_type": "PUT_END", - "key": "object_key_123", - "payload": "optional_payload", - "checksum": 1234567890, - "prefix_hash": 987654321, - "key_sequence_id": 5 -} -``` - -### 3.5 Ordering Guarantee Mechanism - -#### 3.5.1 Global Sequence Number (sequence_id) - -- **Purpose**: Guarantee global order of all OpLog events -- **Generation**: Generated globally incrementally by Primary's `OpLogManager` -- **Check**: Standby checks if sequence_id is continuous - -#### 3.5.2 Key-Level Sequence Number (key_sequence_id) - -- **Purpose**: Guarantee operation order for the same key -- **Generation**: Incremented separately for each key on Primary side -- **Check**: Standby checks if key_sequence_id is increasing - -#### 3.5.3 Out-of-Order Handling - -When key_sequence_id out-of-order is detected: -1. **Rollback**: Delete all state of the key from metadata_store -2. **Replay**: Re-read all OpLog from etcd starting from the key's first sequence_id -3. **Rewrite**: Re-apply all OpLog in correct order to rebuild metadata - -For detailed design, please refer to: `doc/en/rfc-oplog-rollback-replay-on-sequence-violation.md` - -### 3.6 Snapshot Integration - -#### 3.6.1 Record Sequence ID During Snapshot - -- When snapshot is generated, record current OpLog sequence_id -- Write snapshot info to etcd: `mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` - -#### 3.6.2 OpLog Cleanup - -- After snapshot generation, OpLog before snapshot can be cleaned up -- Cleanup strategy: Query minimum existing sequence_id from etcd, use DeleteRange to delete - -For detailed design, please refer to: `doc/en/rfc-oplog-cleanup-start-sequence-id.md` - -### 3.7 Standby Service Integration - -#### 3.7.1 Problem - -In existing code, Standby only blocks and waits during leader election, without running Standby service to synchronize OpLog. - -#### 3.7.2 Solution - -In `MasterServiceSupervisor::Start()`: -1. Check if there is currently a leader -2. If there is a leader and it's not self → Start Standby service (watch OpLog and apply) -3. After successful election → Stop Standby service and promote to Primary - -For detailed design, please refer to: `doc/en/rfc-standby-service-integration.md` - -### 3.8 Lease Initialization When Standby Promotes to Primary - -#### 3.8.1 Problem - -Objects on Standby all have lease = 0 (because OpLog only contains PUT_END, not renewal information), and all objects will expire immediately after promotion to Primary. - -#### 3.8.2 Solution - -In `HotStandbyService::Promote()`: -1. Stop Standby service -2. Iterate through all metadata -3. For objects with lease_timeout = 0, grant default lease time (`default_kv_lease_ttl`) - -For detailed design, please refer to: `doc/en/rfc-standby-promotion-lease-initialization.md` - -### 3.9 Memory Optimization: key_sequence_map_ Cleanup - -#### 3.9.1 Problem - -`key_sequence_map_` on Standby side is used to track `key_sequence_id` for each key. After metadata is deleted, these entries are still retained, which may cause memory leaks during long-term operation. - -#### 3.9.2 Solution - -Implement periodic cleanup mechanism: -- **Cleanup Condition**: Last operation is `REMOVE` and more than 1 hour has passed -- **Cleanup Frequency**: Scan once per hour -- **Retention Strategy**: Keys with `PUT_END` and `PUT_REVOKE` operations are not cleaned - -For detailed design, please refer to: `doc/en/rfc-oplog-key-sequence-map-cleanup.md` - -## 4. Key Design Points Summary - -1. **etcd as Intermediate Storage**: Leverage etcd's strong consistency and Watch mechanism -2. **Record Only Critical Operations**: PUT_END, PUT_REVOKE, REMOVE, do not record LEASE_RENEW -3. **Dual Sequence Number Guarantee**: Global sequence_id + key-level key_sequence_id -4. **Snapshot Integration**: Integrate with existing snapshot mechanism, support OpLog cleanup -5. **Standby Service Runs in Parallel**: Continuously synchronize data during election waiting period -6. **Memory Optimization**: Periodically clean up expired entries in key_sequence_map_ - diff --git a/doc/zh/diagrams/etcd-hot-standby-architecture.puml b/doc/zh/diagrams/etcd-hot-standby-architecture.puml deleted file mode 100644 index 1b27d929ed..0000000000 --- a/doc/zh/diagrams/etcd-hot-standby-architecture.puml +++ /dev/null @@ -1,139 +0,0 @@ -@startuml etcd-hot-standby-architecture -!theme plain -skinparam componentStyle rectangle -skinparam linetype ortho - -title etcd热备架构图 - -package "Primary Master" { - component [MasterService] as MasterService { - + AppendOpLogAndNotify() - + SerializeMetadataForOpLog() - + RestoreFromStandbySnapshot() - } - - component [OpLogManager] as OpLogManager { - + Append() - + SetEtcdOpLogStore() - + SetInitialSequenceId() - } - - component [EtcdOpLogStore] as EtcdOpLogStore { - + WriteOpLog() - + UpdateLatestSequenceId() - + CleanupOpLogBefore() - } -} - -package "Standby Master" { - component [HotStandbyService] as HotStandbyService { - + Start() - + Stop() - + Promote() - + GetSyncStatus() - + ExportMetadataSnapshot() - } - - component [OpLogWatcher] as OpLogWatcher { - + StartFromSequenceId() - + WatchOpLog() - + ReadOpLogSinceWithRevision() - } - - component [OpLogApplier] as OpLogApplier { - + ApplyOpLogEntry() - + ApplyOpLogEntries() - + ProcessPendingEntries() - + RequestMissingOpLog() - } - - component [StandbyMetadataStore] as StandbyMetadataStore { - + PutMetadata() - + Remove() - + Snapshot() - } -} - -package "协调组件" { - component [MasterServiceSupervisor] as Supervisor { - + Start() - + StartStandbyService() - } - - component [MasterViewHelper] as ViewHelper { - + ElectLeader() - + KeepLeader() - } - - component [EtcdHelper] as EtcdHelper { - + ConnectToEtcdStoreClient() - + GetRangeAsJson() - + WatchWithPrefixFromRevisionV2() - + GrantLease() - + CreateWithLease() - } -} - -cloud "etcd" { - database "OpLog Storage" as OpLogStorage { - + /oplog/{cluster_id}/{sequence_id} - + /oplog/{cluster_id}/latest - } - - database "Leader Election" as LeaderElection { - + /mooncake-store/{cluster_id}/master_view - } -} - -' Primary Master 内部关系 -MasterService --> OpLogManager : 生成OpLog -OpLogManager --> EtcdOpLogStore : 写入etcd -EtcdOpLogStore --> OpLogStorage : 存储OpLog - -' Standby Master 内部关系 -HotStandbyService --> OpLogWatcher : 启动watch -HotStandbyService --> OpLogApplier : 应用OpLog -HotStandbyService --> StandbyMetadataStore : 存储metadata -OpLogWatcher --> OpLogApplier : 转发OpLog事件 -OpLogApplier --> StandbyMetadataStore : 更新metadata - -' Standby 与 etcd 关系 -OpLogWatcher --> OpLogStorage : Watch + Read -OpLogApplier --> OpLogStorage : 请求缺失OpLog - -' 协调组件关系 -Supervisor --> ViewHelper : Leader选举 -Supervisor --> MasterService : 启动Primary -Supervisor --> HotStandbyService : 启动Standby -ViewHelper --> LeaderElection : 选举Leader -ViewHelper --> EtcdHelper : etcd操作 -EtcdOpLogStore --> EtcdHelper : etcd操作 -OpLogWatcher --> EtcdHelper : etcd操作 - -' 故障切换流程 -HotStandbyService ..> Supervisor : Promote()后返回metadata -Supervisor ..> MasterService : RestoreFromStandbySnapshot() - -note right of OpLogStorage - OpLog存储格式: - Key: /oplog/{cluster_id}/{sequence_id} - Value: JSON序列化的OpLogEntry - 包含: op_type, object_key, payload等 -end note - -note right of LeaderElection - Leader选举: - - 使用etcd lease机制 - - TTL: 5秒 - - 通过CreateWithLease竞争 -end note - -note bottom of OpLogApplier - 顺序保证: - - 使用全局sequence_id保证顺序 - - 乱序的OpLog会进入pending队列 - - 缺失的OpLog会从etcd请求 -end note - -@enduml - diff --git a/doc/zh/diagrams/etcd-hot-standby-diagrams-README.md b/doc/zh/diagrams/etcd-hot-standby-diagrams-README.md deleted file mode 100644 index b970339898..0000000000 --- a/doc/zh/diagrams/etcd-hot-standby-diagrams-README.md +++ /dev/null @@ -1,127 +0,0 @@ -# etcd热备架构图表说明 - -本文档包含基于当前代码实现的etcd热备架构的PlantUML图表。 - -## 图表文件 - -### 1. `etcd-hot-standby-architecture.puml` - 整体架构图 - -展示了etcd热备系统的整体架构，包括： - -- **Primary Master组件**: - - `MasterService`: 核心服务，处理客户端请求 - - `OpLogManager`: 生成和管理OpLog - - `EtcdOpLogStore`: 将OpLog写入etcd - -- **Standby Master组件**: - - `HotStandbyService`: Standby服务主控制器 - - `OpLogWatcher`: 从etcd监听OpLog变化 - - `OpLogApplier`: 应用OpLog到本地metadata store - - `StandbyMetadataStore`: Standby的metadata存储 - -- **协调组件**: - - `MasterServiceSupervisor`: 管理Primary/Standby切换 - - `MasterViewHelper`: 处理Leader选举 - - `EtcdHelper`: etcd操作的C++ wrapper - -- **etcd存储**: - - OpLog存储: `/oplog/{cluster_id}/{sequence_id}` - - Leader选举: `/mooncake-store/{cluster_id}/master_view` - -### 2. `etcd-hot-standby-sequence.puml` - 时序图 - -展示了关键流程的时序关系，包括： - -1. **Primary启动流程**: - - Leader选举 - - MasterService初始化 - - OpLogManager设置EtcdOpLogStore - -2. **Standby启动流程**: - - 检测已有Leader - - 热启动 vs 冷启动 - - 快照加载(可选) - - 历史OpLog读取 - - Watch启动 - -3. **写入操作流程**: - - 客户端写入请求 - - Primary生成OpLog - - 写入etcd - - Standby接收并应用 - -4. **故障切换流程**: - - Leader lease过期检测 - - Standby最终同步 - - 重新选举 - - 新Primary初始化 - -### 3. `etcd-hot-standby-flow.puml` - 流程图 - -展示了数据流和控制流，包括： - -1. **OpLog写入流程**: Primary如何生成和写入OpLog -2. **Standby同步流程**: Standby如何启动和同步数据 -3. **OpLog应用流程**: Standby如何应用OpLog(包括乱序处理) -4. **故障切换流程**: Standby如何提升为Primary -5. **OpLog清理流程**: 如何清理etcd中的旧OpLog -6. **批量更新流程**: latest_sequence_id的批量更新机制 - -## 关键设计点 - -### 1. 顺序保证 -- 使用全局`sequence_id`保证OpLog顺序 -- Standby通过`expected_sequence_id`检测乱序 -- 乱序的OpLog进入`pending_entries_`队列 -- 缺失的OpLog从etcd主动请求 - -### 2. 一致性保证 -- 使用etcd revision实现"read then watch"的一致性 -- `ReadOpLogSinceWithRevision`返回revision -- Watch从`revision + 1`开始，确保不丢失事件 - -### 3. 性能优化 -- `latest_sequence_id`批量更新(每100条或每1秒) -- OpLog读取使用分页(每批1000条) -- 使用固定宽度sequence_id确保etcd key的字典序 - -### 4. 故障恢复 -- Standby提升前进行最终同步 -- 新Primary从Standby的metadata快照恢复 -- OpLog sequence_id连续，避免回退 - -## 使用方法 - -### 查看图表 - -1. **在线查看**: 使用PlantUML在线服务器 - - 访问: http://www.plantuml.com/plantuml/uml/ - - 复制`.puml`文件内容粘贴查看 - -2. **VS Code插件**: 安装PlantUML插件 - - 插件: `PlantUML` - - 打开`.puml`文件，按`Alt+D`预览 - -3. **命令行工具**: 使用PlantUML命令行工具 - ```bash - java -jar plantuml.jar etcd-hot-standby-architecture.puml - ``` - -### 导出图片 - -```bash -# 导出为PNG -java -jar plantuml.jar -tpng *.puml - -# 导出为SVG -java -jar plantuml.jar -tsvg *.puml - -# 导出为PDF -java -jar plantuml.jar -tpdf *.puml -``` - -## 相关文档 - -- [RFC: etcd热备完整方案](../rfc-oplog-hot-standby-complete.md) -- [实现计划](../rfc-oplog-implementation-plan.md) - diff --git a/doc/zh/diagrams/etcd-hot-standby-flow.puml b/doc/zh/diagrams/etcd-hot-standby-flow.puml deleted file mode 100644 index 33de9cb159..0000000000 --- a/doc/zh/diagrams/etcd-hot-standby-flow.puml +++ /dev/null @@ -1,173 +0,0 @@ -@startuml etcd-hot-standby-flow -!theme plain -skinparam activity { - BackgroundColor #E1F5FF - BorderColor #0066CC - FontColor #000000 -} -skinparam arrow { - Color #0066CC - Thickness 2 -} - -title etcd热备数据流和控制流程图 - -partition "OpLog写入流程" { -start -:Primary Master接收操作请求; -:MasterService处理请求; -:序列化metadata为JSON; -:OpLogManager.Append(); -note right - 生成: - - sequence_id (全局递增) - - timestamp_ms - - checksum - - prefix_hash -end note -:EtcdOpLogStore.WriteOpLog(); -:写入etcd: /oplog/{cluster_id}/{sequence_id}; -:触发批量更新latest_sequence_id; -note right - 批量更新策略: - - 每100条或每1秒 - - 减少etcd写入压力 -end note -stop -} - -partition "Standby同步流程" { -start -:Standby启动; -if (已有本地metadata?) then (是 - 热启动) - :读取本地last_seq_id; - :OpLogApplier.Recover(last_seq_id); -else (否 - 冷启动) - if (启用快照?) then (是) - :SnapshotProvider.LoadLatestSnapshot(); - :加载快照到StandbyMetadataStore; - :OpLogApplier.Recover(snapshot_seq_id); - endif -endif -:OpLogWatcher.StartFromSequenceId(); -:读取历史OpLog (ReadOpLogSinceWithRevision); -note right - 使用分页读取: - - 每批1000条 - - 返回etcd revision -end note -:应用OpLog到StandbyMetadataStore; -:设置next_watch_revision = revision + 1; -:启动Watch线程 (WatchWithPrefixFromRevisionV2); -:持续监听etcd OpLog变化; -stop -} - -partition "OpLog应用流程" { -start -:OpLogWatcher收到Watch事件; -:反序列化OpLogEntry; -:OpLogApplier.ApplyOpLogEntry(); -if (sequence_id == expected_sequence_id?) then (是) - :直接应用; - switch (op_type) - case (PUT_END) - :反序列化payload; - :StandbyMetadataStore.PutMetadata(); - case (PUT_REVOKE) - :StandbyMetadataStore.Remove(); - case (REMOVE) - :StandbyMetadataStore.Remove(); - endswitch - :expected_sequence_id++; - :处理pending队列; -else (否 - 乱序) - if (sequence_id < expected_sequence_id?) then (是 - 重复) - :忽略(已处理); - else (否 - 超前) - :加入pending队列; - :记录missing_sequence_ids; - if (等待超过5秒?) then (是) - :RequestMissingOpLog(); - :从etcd读取缺失OpLog; - :应用缺失OpLog; - endif - endif -endif -stop -} - -partition "故障切换流程" { -start -:etcd检测到Leader lease过期; -:MasterViewHelper检测到Leader删除; -:MasterServiceSupervisor触发切换; -:HotStandbyService.Promote(); -:停止OpLogWatcher; -:最终同步: 读取剩余OpLog; -note right - 循环读取直到: - - 没有更多OpLog - - 或读取失败 -end note -:应用所有剩余OpLog; -:ExportMetadataSnapshot(); -:GetLatestAppliedSequenceId(); -:MasterServiceSupervisor重新选举; -if (选举成功?) then (是) - :创建新MasterService; - :OpLogManager.SetInitialSequenceId(last_seq_id); - :MasterService.RestoreFromStandbySnapshot(); - note right - 恢复过程: - - 创建DummyBufferAllocator - - 重建Replica对象 - - 恢复metadata到本地 - - 不恢复lease信息 - end note - :启动新Primary服务; -else (否) - :继续作为Standby; -endif -stop -} - -partition "OpLog清理流程" { -start -:定期触发清理任务; -:EtcdOpLogStore.CleanupOpLogBefore(); -:查询etcd中最小sequence_id; -note right - Scheme 3: - - 不依赖持久化的"cleaned_upto" - - 查询实际最小sequence_id - - 更可靠 -end note -if (最小seq_id < before_sequence_id?) then (是) - :DeleteRange(/oplog/{cluster_id}/0, before_seq_id); - :删除etcd中的旧OpLog; -else (否) - :无需清理; -endif -stop -} - -partition "批量更新latest_sequence_id流程" { -start -:EtcdOpLogStore.WriteOpLog(); -:pending_latest_seq_id = sequence_id; -:pending_count++; -if (pending_count >= 100\n或距离上次更新 >= 1秒?) then (是) - :DoBatchUpdate(); - :UpdateLatestSequenceId(pending_latest_seq_id); - :写入etcd: /oplog/{cluster_id}/latest; - :pending_count = 0; - :last_update_time = now; -else (否) - :继续累积; -endif -stop -} - -@enduml - diff --git a/doc/zh/diagrams/etcd-hot-standby-sequence.puml b/doc/zh/diagrams/etcd-hot-standby-sequence.puml deleted file mode 100644 index ecdd53809f..0000000000 --- a/doc/zh/diagrams/etcd-hot-standby-sequence.puml +++ /dev/null @@ -1,259 +0,0 @@ -@startuml etcd-hot-standby-sequence -!theme plain -skinparam sequenceMessageAlign center -skinparam sequenceArrowThickness 2 - -title etcd热备关键时序图 - -== Primary启动 == - -actor User -participant Supervisor as "MasterServiceSupervisor" -participant ViewHelper as "MasterViewHelper" -participant EtcdHelper as "EtcdHelper" -database etcd as "etcd" -participant MasterService as "MasterService" -participant OpLogManager as "OpLogManager" -participant EtcdOpLogStore as "EtcdOpLogStore" - -User -> Supervisor: 启动服务 -activate Supervisor - -Supervisor -> ViewHelper: ElectLeader() -activate ViewHelper -ViewHelper -> EtcdHelper: GrantLease(TTL=5s) -EtcdHelper -> etcd: 创建lease -etcd --> EtcdHelper: lease_id -ViewHelper -> EtcdHelper: CreateWithLease(key, lease_id) -EtcdHelper -> etcd: 尝试创建leader key -alt 成功 - etcd --> EtcdHelper: 成功，成为Leader - ViewHelper --> Supervisor: 选举成功 -else 失败 - etcd --> EtcdHelper: 失败，已有Leader - ViewHelper -> EtcdHelper: WatchUntilDeleted() - EtcdHelper -> etcd: Watch leader key - etcd --> EtcdHelper: Leader删除事件 - ViewHelper --> Supervisor: Leader已删除，重试选举 -end -deactivate ViewHelper - -Supervisor -> MasterService: 创建MasterService -activate MasterService -MasterService -> OpLogManager: 创建OpLogManager -activate OpLogManager -MasterService -> EtcdOpLogStore: 创建EtcdOpLogStore(enable_batch=true) -activate EtcdOpLogStore -OpLogManager -> EtcdOpLogStore: SetEtcdOpLogStore() -deactivate EtcdOpLogStore -deactivate OpLogManager -deactivate MasterService - -Supervisor -> MasterService: 启动服务 -activate MasterService -MasterService -> ViewHelper: KeepLeader(lease_id) -activate ViewHelper -ViewHelper -> EtcdHelper: KeepAlive(lease_id) -EtcdHelper -> etcd: 定期续约 -deactivate ViewHelper -deactivate MasterService -deactivate Supervisor - -== Standby启动 == - -participant HotStandbyService as "HotStandbyService" -participant OpLogWatcher as "OpLogWatcher" -participant OpLogApplier as "OpLogApplier" -participant StandbyMetadataStore as "StandbyMetadataStore" - -User -> Supervisor: 启动服务(已有Leader) -activate Supervisor - -Supervisor -> ViewHelper: GetMasterView() -activate ViewHelper -ViewHelper -> EtcdHelper: Get(key) -EtcdHelper -> etcd: 查询leader -etcd --> EtcdHelper: 返回leader地址 -EtcdHelper --> ViewHelper: leader地址 -ViewHelper --> Supervisor: 已有Leader -deactivate ViewHelper - -Supervisor -> HotStandbyService: 创建HotStandbyService -activate HotStandbyService -HotStandbyService -> StandbyMetadataStore: 创建StandbyMetadataStore -activate StandbyMetadataStore -HotStandbyService -> OpLogApplier: 创建OpLogApplier -activate OpLogApplier -HotStandbyService -> OpLogWatcher: 创建OpLogWatcher -activate OpLogWatcher - -Supervisor -> HotStandbyService: Start(etcd_endpoints, cluster_id) -HotStandbyService -> EtcdHelper: ConnectToEtcdStoreClient() -EtcdHelper -> etcd: 连接etcd -etcd --> EtcdHelper: 连接成功 - -alt 热启动(已有metadata) - HotStandbyService -> OpLogApplier: GetExpectedSequenceId() - OpLogApplier --> HotStandbyService: last_seq_id - HotStandbyService -> OpLogApplier: Recover(last_seq_id) -else 冷启动(无metadata) - opt 启用快照 - HotStandbyService -> StandbyMetadataStore: LoadLatestSnapshot() - StandbyMetadataStore --> HotStandbyService: snapshot + snapshot_seq_id - HotStandbyService -> OpLogApplier: Recover(snapshot_seq_id) - end -end - -HotStandbyService -> OpLogWatcher: StartFromSequenceId(start_seq_id) -OpLogWatcher -> EtcdOpLogStore: ReadOpLogSinceWithRevision(start_seq_id) -activate EtcdOpLogStore -EtcdOpLogStore -> EtcdHelper: GetRangeAsJson(prefix, limit) -EtcdHelper -> etcd: Range Get -etcd --> EtcdHelper: OpLog entries + revision -EtcdHelper --> EtcdOpLogStore: entries + revision_id -EtcdOpLogStore --> OpLogWatcher: entries + revision_id -deactivate EtcdOpLogStore - -loop 批量读取历史OpLog - OpLogWatcher -> OpLogApplier: ApplyOpLogEntries(batch) - OpLogApplier -> StandbyMetadataStore: PutMetadata() / Remove() - StandbyMetadataStore --> OpLogApplier: 成功 - OpLogApplier --> OpLogWatcher: applied_count -end - -OpLogWatcher -> OpLogWatcher: next_watch_revision = revision_id + 1 -OpLogWatcher -> OpLogWatcher: WatchOpLog() [后台线程] -OpLogWatcher -> EtcdHelper: WatchWithPrefixFromRevisionV2(prefix, start_revision) -EtcdHelper -> etcd: Watch from revision -etcd --> EtcdHelper: OpLog事件流 -deactivate OpLogWatcher -deactivate OpLogApplier -deactivate StandbyMetadataStore -deactivate HotStandbyService -deactivate Supervisor - -== 写入操作流程 == - -participant Client - -Client -> MasterService: PutEnd(key, metadata) -activate MasterService -MasterService -> MasterService: 更新本地metadata -MasterService -> MasterService: SerializeMetadataForOpLog() -MasterService -> OpLogManager: Append(PUT_END, key, payload) -activate OpLogManager -OpLogManager -> OpLogManager: 生成sequence_id -OpLogManager -> OpLogManager: 计算checksum和prefix_hash -OpLogManager -> EtcdOpLogStore: WriteOpLog(entry) -activate EtcdOpLogStore -EtcdOpLogStore -> EtcdHelper: Put(key, value) -EtcdHelper -> etcd: 写入OpLog -etcd --> EtcdHelper: 成功 -EtcdHelper --> EtcdOpLogStore: 成功 -EtcdOpLogStore -> EtcdOpLogStore: TriggerBatchUpdateIfNeeded() -deactivate EtcdOpLogStore -OpLogManager --> MasterService: sequence_id -MasterService --> Client: 成功 -deactivate MasterService -deactivate OpLogManager - -' Standby接收OpLog -etcd -> OpLogWatcher: Watch事件(PUT) -activate OpLogWatcher -OpLogWatcher -> OpLogWatcher: DeserializeOpLogEntry() -OpLogWatcher -> OpLogApplier: ApplyOpLogEntry(entry) -activate OpLogApplier -OpLogApplier -> OpLogApplier: CheckSequenceOrder() -alt 顺序正确 - OpLogApplier -> OpLogApplier: ApplyPutEnd() - OpLogApplier -> StandbyMetadataStore: PutMetadata(key, metadata) - activate StandbyMetadataStore - StandbyMetadataStore --> OpLogApplier: 成功 - deactivate StandbyMetadataStore - OpLogApplier --> OpLogWatcher: 成功 -else 顺序错误(乱序) - OpLogApplier -> OpLogApplier: 加入pending队列 - OpLogApplier -> OpLogApplier: RequestMissingOpLog() - OpLogApplier -> EtcdOpLogStore: ReadOpLog(missing_seq_id) - activate EtcdOpLogStore - EtcdOpLogStore -> EtcdHelper: Get(key) - EtcdHelper -> etcd: 查询OpLog - etcd --> EtcdHelper: OpLog entry - EtcdHelper --> EtcdOpLogStore: entry - EtcdOpLogStore --> OpLogApplier: entry - deactivate EtcdOpLogStore - OpLogApplier -> OpLogApplier: ProcessPendingEntries() -end -deactivate OpLogApplier -deactivate OpLogWatcher - -== 故障切换流程 == - -participant NewPrimary as "New Primary\n(MasterService)" - -etcd -> ViewHelper: Leader lease过期 -activate ViewHelper -ViewHelper -> Supervisor: Leader已删除 -activate Supervisor - -Supervisor -> HotStandbyService: Promote() -activate HotStandbyService -HotStandbyService -> OpLogWatcher: Stop() -activate OpLogWatcher -OpLogWatcher --> HotStandbyService: 已停止 -deactivate OpLogWatcher - -HotStandbyService -> EtcdOpLogStore: ReadOpLogSince(last_seq_id) -activate EtcdOpLogStore -EtcdOpLogStore -> EtcdHelper: GetRangeAsJson() -EtcdHelper -> etcd: Range Get -etcd --> EtcdHelper: 剩余OpLog entries -EtcdHelper --> EtcdOpLogStore: entries -EtcdOpLogStore --> HotStandbyService: entries -deactivate EtcdOpLogStore - -loop 最终同步 - HotStandbyService -> OpLogApplier: ApplyOpLogEntries(batch) - activate OpLogApplier - OpLogApplier -> StandbyMetadataStore: 应用OpLog - StandbyMetadataStore --> OpLogApplier: 成功 - OpLogApplier --> HotStandbyService: applied_count - deactivate OpLogApplier -end - -HotStandbyService -> HotStandbyService: ExportMetadataSnapshot() -HotStandbyService -> HotStandbyService: GetLatestAppliedSequenceId() -HotStandbyService --> Supervisor: snapshot + last_seq_id -deactivate HotStandbyService - -Supervisor -> ViewHelper: ElectLeader() [重新选举] -activate ViewHelper -ViewHelper -> EtcdHelper: GrantLease() + CreateWithLease() -EtcdHelper -> etcd: 选举Leader -etcd --> EtcdHelper: 选举成功 -EtcdHelper --> ViewHelper: 成为新Leader -ViewHelper --> Supervisor: 选举成功 -deactivate ViewHelper - -Supervisor -> NewPrimary: 创建MasterService -activate NewPrimary -NewPrimary -> OpLogManager: SetInitialSequenceId(last_seq_id) -activate OpLogManager -OpLogManager --> NewPrimary: 已设置 -deactivate OpLogManager -NewPrimary -> NewPrimary: RestoreFromStandbySnapshot(snapshot) -NewPrimary -> NewPrimary: 恢复metadata到本地 -NewPrimary --> Supervisor: 恢复完成 -deactivate NewPrimary - -Supervisor -> NewPrimary: 启动服务 -activate NewPrimary -NewPrimary -> ViewHelper: KeepLeader(lease_id) -activate ViewHelper -ViewHelper -> EtcdHelper: KeepAlive(lease_id) -deactivate ViewHelper -deactivate NewPrimary -deactivate Supervisor - -@enduml - diff --git a/doc/zh/diagrams/mooncake-transfer-flow.puml b/doc/zh/diagrams/mooncake-transfer-flow.puml deleted file mode 100644 index 10f41de91a..0000000000 --- a/doc/zh/diagrams/mooncake-transfer-flow.puml +++ /dev/null @@ -1,80 +0,0 @@ -@startuml Mooncake Store 数据传输流程 - -!theme plain -skinparam backgroundColor #FFFFFF -skinparam activity { - BackgroundColor #E8F4F8 - BorderColor #4A90E2 - FontColor #000000 -} -skinparam arrow { - Color #4A90E2 -} - -title Mooncake Store 数据传输流程 - -start - -:TransferSubmitter 接收传输请求\n(Replica Descriptor, Slices); - -:从 Master Service 获取\nReplica Descriptor; - -note right - **Replica Descriptor** 包含： - - transport_endpoint: 目标端点 - - buffer_address: 内存地址 - - size: 数据大小 -end note - -:调用 selectStrategy()\n选择传输策略; - -if (是否为本地传输?) then (是) - :执行 LOCAL_MEMCPY\n本地内存拷贝; - note right - 源和目标在同一进程 - 直接 memcpy - end note - :返回成功; - stop -else (否) - :创建 TransferEngine 传输请求; - - if (传输协议选择) then (RDMA) - :初始化 RDMA Transport; - :建立 RDMA 连接; - :执行 RDMA Write/Read\n零拷贝传输; - note right - **RDMA 优势**： - - 零拷贝，绕过内核 - - 低延迟 - - 高带宽 - end note - :数据直接写入目标 Segment\nAllocatedBuffer; - else (TCP) - :初始化 TCP Transport; - :建立 TCP 连接; - :执行 TCP Write/Read\n标准网络传输; - note right - **TCP 传输**： - - 标准网络协议 - - 兼容性好 - - 需要内核参与 - end note - :数据写入目标 Segment\nAllocatedBuffer; - endif - - :等待传输完成; - - if (传输是否成功?) then (是) - :更新传输指标; - :返回成功; - else (否) - :记录错误日志; - :返回失败; - endif - - stop -endif - -@enduml - diff --git a/doc/zh/diagrams/oplog-data-flow.puml b/doc/zh/diagrams/oplog-data-flow.puml deleted file mode 100644 index 5f5b1175a9..0000000000 --- a/doc/zh/diagrams/oplog-data-flow.puml +++ /dev/null @@ -1,58 +0,0 @@ -@startuml oplog-data-flow -!theme plain -skinparam backgroundColor #FFFFFF -skinparam sequenceMessageAlign center - -actor Client -participant "Primary Master" as Primary -participant "OpLogManager" as OplogMgr -participant "EtcdOpLogStore" as EtcdStore -database etcd -participant "OpLogWatcher" as Watcher -participant "OpLogApplier" as Applier -participant "Standby Master" as Standby - -== PUT_END 操作流程 == - -Client -> Primary: PutEnd(key, ...) -activate Primary -Primary -> OplogMgr: Append(PUT_END, key) -activate OplogMgr -OplogMgr -> OplogMgr: 生成 sequence_id\n生成 key_sequence_id -OplogMgr -> EtcdStore: WriteOpLog(entry) -activate EtcdStore -EtcdStore -> etcd: PUT /oplog/{seq} -activate etcd -etcd --> EtcdStore: Success -deactivate etcd -EtcdStore --> OplogMgr: Success -deactivate EtcdStore -OplogMgr --> Primary: sequence_id -deactivate OplogMgr -Primary --> Client: Success -deactivate Primary - -== Standby 同步流程 == - -etcd -> Watcher: Watch Event (新 OpLog) -activate Watcher -Watcher -> Applier: ApplyOpLogEntry(entry) -activate Applier -Applier -> Applier: CheckSequenceOrder() -alt 顺序正确 - Applier -> Standby: UpdateMetadata(key, ...) - activate Standby - Standby --> Applier: Success - deactivate Standby -else 顺序错误 - Applier -> Applier: RollbackAndReplay() - Applier -> etcd: ReadOpLogForKey() - etcd --> Applier: OpLog Entries - Applier -> Applier: Replay OpLog -end -Applier --> Watcher: Success -deactivate Applier -deactivate Watcher - -@enduml - diff --git a/doc/zh/diagrams/oplog-failover-sequence.puml b/doc/zh/diagrams/oplog-failover-sequence.puml deleted file mode 100644 index 5945ecbb10..0000000000 --- a/doc/zh/diagrams/oplog-failover-sequence.puml +++ /dev/null @@ -1,66 +0,0 @@ -@startuml oplog-failover-sequence -!theme plain -skinparam backgroundColor #FFFFFF -skinparam sequenceMessageAlign center - -participant "Primary Master" as Primary -database etcd -participant "Standby Master" as Standby -participant "MasterServiceSupervisor" as Supervisor -participant "HotStandbyService" as HotStandby -participant "OpLogWatcher" as Watcher -participant "OpLogApplier" as Applier - -== 正常运行阶段 == - -Primary -> etcd: KeepAlive Lease -etcd -> Supervisor: Watch Leader (存在) -activate Supervisor -Supervisor -> HotStandby: StartStandby() -activate HotStandby -HotStandby -> Watcher: Start() -activate Watcher -Watcher -> etcd: Watch OpLog -etcd -> Watcher: OpLog Events -Watcher -> Applier: ApplyOpLogEntry() -activate Applier -Applier -> Standby: UpdateMetadata() -deactivate Applier -deactivate Watcher -deactivate HotStandby -deactivate Supervisor - -== Primary 故障 == - -Primary -x etcd: Lease Expired (故障) -etcd -> Supervisor: Leader Deleted Event -activate Supervisor -Supervisor -> HotStandby: Stop() -activate HotStandby -HotStandby -> Watcher: Stop() -deactivate Watcher -deactivate HotStandby - -== Standby 提升为 Primary == - -Supervisor -> Standby: Promote() -activate Standby -Standby -> Standby: 初始化 Lease\n清理过期 metadata -Standby -> etcd: ElectLeader() -activate etcd -etcd -> Standby: Leader Elected -deactivate etcd -Standby -> Supervisor: Primary Mode -deactivate Standby -deactivate Supervisor - -note over Standby - 1. 停止 Standby 服务 - 2. 遍历所有 metadata - 3. 对 lease=0 的对象授予默认租约 - 4. 执行一次完整的 metadata 清理 - 5. 开始 Leader 选举 -end note - -@enduml - diff --git a/doc/zh/diagrams/oplog-hot-standby-architecture.puml b/doc/zh/diagrams/oplog-hot-standby-architecture.puml deleted file mode 100644 index d8ee863650..0000000000 --- a/doc/zh/diagrams/oplog-hot-standby-architecture.puml +++ /dev/null @@ -1,65 +0,0 @@ -@startuml oplog-hot-standby-architecture -!theme plain -skinparam backgroundColor #FFFFFF -skinparam componentStyle rectangle -skinparam defaultFontSize 12 - -package "Primary Master" #E8F4F8 { - component [MasterService] as MasterService - component [OpLogManager] as OpLogManager - component [EtcdOpLogStore] as EtcdOpLogStore - - MasterService --> OpLogManager : 记录操作 - OpLogManager --> EtcdOpLogStore : 写入 OpLog -} - -package "etcd Cluster" #FFF4E6 { - database [etcd] as etcd -} - -package "Standby Master" #F0F8E8 { - component [MasterServiceSupervisor] as Supervisor - component [HotStandbyService] as HotStandby - component [OpLogWatcher] as Watcher - component [OpLogApplier] as Applier - component [MetadataStore] as MetadataStore - - Supervisor --> HotStandby : 启动/停止 - HotStandby --> Watcher : Watch OpLog - Watcher --> Applier : 应用 OpLog - Applier --> MetadataStore : 更新 metadata -} - -EtcdOpLogStore --> etcd : 写入 OpLog -etcd --> Watcher : Watch 事件 - -note right of OpLogManager - **职责**： - - 生成 sequence_id - - 生成 key_sequence_id - - 维护内存缓冲区 -end note - -note right of Applier - **职责**： - - 检查顺序 - - 处理乱序 - - 清理过期条目 -end note - -note right of MasterService - **操作**： - - PutEnd() - - Remove() - - Eviction() -end note - -note right of etcd - **Key 设计**： - - /oplog/{cluster_id}/{sequence_id} - - /oplog/{cluster_id}/latest - - Watch API -end note - -@enduml - diff --git a/doc/zh/diagrams/oplog-hot-standby-complete-architecture.puml b/doc/zh/diagrams/oplog-hot-standby-complete-architecture.puml deleted file mode 100644 index e34f773607..0000000000 --- a/doc/zh/diagrams/oplog-hot-standby-complete-architecture.puml +++ /dev/null @@ -1,123 +0,0 @@ -@startuml oplog-hot-standby-complete-architecture -!theme plain -skinparam backgroundColor #FFFFFF -skinparam componentStyle rectangle -skinparam defaultFontSize 11 - -package "Master Cluster (HA)" { - - package "Primary Master (Leader)" #90EE90 { - component [MasterService\nMetadata Management] as MasterService - component [OpLogManager\nGenerate & Buffer] as OpLogManager - component [EtcdOpLogStore\nWrite to etcd] as EtcdOpLogStore - - MasterService --> OpLogManager : Step 1:\nWrite op generates OpLog - OpLogManager --> EtcdOpLogStore : Step 2:\nWrite OpLog to etcd - } - - package "Standby Master 1 (Hot Standby)" #FFA500 { - component [MasterServiceSupervisor\nLifecycle Manager] as Supervisor1 - component [HotStandbyService\nCore Service] as HotStandby1 - component [OpLogWatcher\nWatch etcd] as OpLogWatcher1 - component [OpLogApplier\nApply Changes] as OpLogApplier1 - component [MetadataStore\nReplica Data] as MetadataStore1 - - Supervisor1 --> HotStandby1 : Start/Stop\nStandby mode - HotStandby1 --> OpLogWatcher1 : Step 3:\nWatch OpLog - OpLogWatcher1 --> OpLogApplier1 : Step 4:\nForward OpLog - OpLogApplier1 --> MetadataStore1 : Step 5:\nApply changes - } - - package "Standby Master 2 (Hot Standby)" #FFA500 { - component [MasterServiceSupervisor\nLifecycle Manager] as Supervisor2 - component [HotStandbyService\nCore Service] as HotStandby2 - component [OpLogWatcher\nWatch etcd] as OpLogWatcher2 - component [OpLogApplier\nApply Changes] as OpLogApplier2 - component [MetadataStore\nReplica Data] as MetadataStore2 - - Supervisor2 --> HotStandby2 : Start/Stop\nStandby mode - HotStandby2 --> OpLogWatcher2 : Watch OpLog - OpLogWatcher2 --> OpLogApplier2 : Forward OpLog - OpLogApplier2 --> MetadataStore2 : Apply changes - } -} - -package "vLLM Inference Cluster" #ADD8E6 { - component [vLLM Instance 1] as vLLM1 - component [vLLM Instance 2] as vLLM2 - component [vLLM Instance N] as vLLMN -} - -package "etcd Cluster" #DDA0DD { - database [Service Discovery\n/mooncake/master/view] as ServiceDiscovery - database [Leader Election\n/mooncake/master/leader] as LeaderElection - database [OpLog Storage\n/oplog/{cluster_id}/{sequence_id}] as OpLogStorage -} - -' Primary interactions -MasterService <--> vLLM1 : RPC\n(Query/Put/Remove) -MasterService <--> vLLM2 : RPC\n(Query/Put/Remove) -MasterService <--> vLLMN : RPC\n(Query/Put/Remove) - -' etcd interactions - OpLog -EtcdOpLogStore --> OpLogStorage : Write OpLog\n(sequence_id) -OpLogStorage --> OpLogWatcher1 : Watch Events\n(Real-time sync) -OpLogStorage --> OpLogWatcher2 : Watch Events\n(Real-time sync) - -' etcd interactions - Leader Election -MasterService --> LeaderElection : Lease KeepAlive\n(TTL=5s) -Supervisor1 --> LeaderElection : Watch Leader Key -Supervisor2 --> LeaderElection : Watch Leader Key - -note right of MasterService - **Primary Responsibilities:** - - Handle all client requests - - Generate OpLog for writes - - Write OpLog to etcd - - Manage metadata -end note - -note right of HotStandby1 - **Standby Responsibilities:** - - Watch OpLog from etcd - - Apply OpLog to metadata - - Maintain replica metadata - - Ready for promotion -end note - -note right of OpLogManager - **OpLogManager:** - - Generate sequence_id - - Generate key_sequence_id - - Maintain buffer -end note - -note right of OpLogApplier1 - **OpLogApplier:** - - Check sequence order - - Handle out-of-order - - Cleanup stale entries -end note - -note right of OpLogStorage - **etcd OpLog Key:** - - /oplog/{cluster_id}/{sequence_id} - - /oplog/{cluster_id}/latest - - Watch API -end note - -note right of LeaderElection - **etcd Services:** - - Service Discovery - - Leader Election - - Lease Management -end note - -legend right - |<#90EE90> **Green (Primary)** | Active leader handling requests | - |<#FFA500> **Orange (Standby)** | Hot standby with replica data | - |<#DDA0DD> **Purple (etcd)** | Coordination & OpLog storage | - |<#ADD8E6> **Light Blue (Clients)** | vLLM inference instances | -endlegend - -@enduml diff --git a/doc/zh/rfc-batched-delete-events-via-etcd.md b/doc/zh/rfc-batched-delete-events-via-etcd.md deleted file mode 100644 index 8c0196b0b5..0000000000 --- a/doc/zh/rfc-batched-delete-events-via-etcd.md +++ /dev/null @@ -1,455 +0,0 @@ -# 基于 etcd 批量压缩 Delete 事件方案 - -## 问题背景 - -### 当前设计回顾 - -根据之前的分析： -1. **驱逐事件频率极高**：可达 130,000 次/秒 -2. **当前方案**： - - 显式 Delete 事件 → 写入 etcd（强一致性） - - 驱逐产生的 Delete 事件 → 不写入 etcd（由 Standby 自己根据租约到期决定） - -### 新方案需求 - -用户提出：使用 etcd 作为中间媒介，对驱逐产生的 delete 事件进行**批量压缩组装**后写入 etcd，而不是每次驱逐都写入一次。 - -## 方案设计 - -### 1. 架构设计 - -``` -┌─────────────────────────────────────────────────────────┐ -│ Primary Master │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ │ -│ │ Eviction │ │ Delete │ │ -│ │ Thread │ │ Event │ │ -│ │ │ │ Buffer │ │ -│ ┌──────────────┘ ┌──────────────┘ │ -│ │ │ │ -│ │ 驱逐事件 │ 显式 Delete │ -│ ▼ ▼ │ -│ ┌──────────────────────────────────────┐ │ -│ │ BatchedDeleteEventManager │ │ -│ │ - 批量收集 delete 事件 │ │ -│ │ - 压缩/去重 │ │ -│ │ - 定时批量写入 etcd │ │ -│ └──────────────────────────────────────┘ │ -│ │ │ -│ │ 批量写入 │ -│ ▼ │ -│ ┌──────────────┐ │ -│ │ etcd │ │ -│ └──────────────┘ │ -└─────────────────────────────────────────────────────────┘ - │ - │ Watch - ▼ -┌─────────────────────────────────────────────────────────┐ -│ Standby Masters │ -│ ┌──────────────────────────────────────┐ │ -│ │ DeleteEventWatcher │ │ -│ │ - Watch etcd delete events │ │ -│ │ - 解压缩/应用 delete 事件 │ │ -│ └──────────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────┘ -``` - -### 2. 批量压缩策略 - -#### 方案 A：时间窗口批量（推荐） - -**原理**： -- 收集固定时间窗口内的所有 delete 事件（如 1 秒） -- 时间窗口到期后，批量写入 etcd -- 使用压缩格式减少数据量 - -**优点**： -- 简单易实现 -- 延迟可控（最多 1 秒） -- 批量写入减少 etcd 压力 - -**缺点**： -- 固定延迟（1 秒） -- 如果事件很少，也会等待 1 秒 - -#### 方案 B：大小阈值批量 - -**原理**： -- 收集 delete 事件直到达到阈值（如 1000 条） -- 达到阈值后立即批量写入 -- 同时设置最大等待时间（如 1 秒） - -**优点**： -- 高吞吐时延迟低（立即写入） -- 低吞吐时延迟可控（最多 1 秒） - -**缺点**： -- 实现稍复杂 -- 需要同时考虑大小和时间两个维度 - -#### 方案 C：混合策略（推荐） - -**原理**： -- 同时设置大小阈值（如 1000 条）和时间窗口（如 1 秒） -- 满足任一条件即批量写入 -- 使用压缩格式减少数据量 - -**优点**： -- 兼顾性能和延迟 -- 高吞吐时立即写入，低吞吐时定时写入 - -### 3. 压缩格式设计 - -#### 格式 A：JSON 数组（简单） - -```json -{ - "batch_id": "2024-01-01T12:00:00.000Z", - "timestamp": 1704110400000, - "keys": [ - "key1", "key2", "key3", ... - ], - "count": 1000 -} -``` - -**优点**： -- 简单易实现 -- 易于调试 - -**缺点**： -- 数据量大（每个 key 都是完整字符串） -- etcd value 大小限制（1.5MB） - -#### 格式 B：前缀压缩（推荐） - -```json -{ - "batch_id": "2024-01-01T12:00:00.000Z", - "timestamp": 1704110400000, - "compressed": true, - "format": "prefix_tree", - "data": { - "prefix1": ["suffix1", "suffix2", ...], - "prefix2": ["suffix3", "suffix4", ...], - ... - }, - "count": 1000 -} -``` - -**优点**： -- 压缩率高（如果 key 有共同前缀） -- 减少 etcd value 大小 - -**缺点**： -- 实现复杂 -- 如果 key 没有共同前缀，压缩效果差 - -#### 格式 C：Bloom Filter + Key List（推荐用于大量 key） - -**原理**： -- 使用 Bloom Filter 快速判断 key 是否存在 -- 对于少量 key，直接存储完整列表 -- 对于大量 key，使用 Bloom Filter + 采样 - -```json -{ - "batch_id": "2024-01-01T12:00:00.000Z", - "timestamp": 1704110400000, - "count": 10000, - "bloom_filter": "base64_encoded_bloom_filter", - "sample_keys": ["key1", "key2", ...], // 前 100 个 key 作为样本 - "hash_prefix": "abc123" // 如果 key 有 hash 前缀，可以进一步压缩 -} -``` - -**优点**： -- 压缩率极高（Bloom Filter 很小） -- 适合大量 key 的场景 - -**缺点**： -- 有误判率（Bloom Filter 特性） -- 需要额外存储完整 key 列表用于精确匹配 - -#### 格式 D：简单列表 + 压缩（推荐用于中等数量 key） - -```json -{ - "batch_id": "2024-01-01T12:00:00.000Z", - "timestamp": 1704110400000, - "keys": ["key1", "key2", ...], // 最多 1000 条 - "count": 1000 -} -``` - -**优点**： -- 简单直接 -- 无压缩开销 -- 易于解析和应用 - -**缺点**： -- 如果 key 很长，数据量大 -- 受 etcd value 大小限制 - -### 4. etcd Key 设计 - -#### 方案 A：单个 Key + 版本号 - -``` -mooncake-store/deletes/batch/{batch_id} -``` - -**优点**： -- 简单 -- 易于 Watch - -**缺点**： -- 如果批量很大，单个 value 可能超过 etcd 限制（1.5MB） -- 需要处理 value 大小限制 - -#### 方案 B：分片 Key（推荐） - -``` -mooncake-store/deletes/batch/{batch_id}/shard/{shard_id} -``` - -**原理**： -- 将大批量分成多个 shard（每个 shard 最多 1000 条 key） -- 每个 shard 写入一个 etcd key -- 使用事务保证原子性 - -**优点**： -- 避免单个 value 过大 -- 可以并行写入多个 shard -- 易于 Watch 和解析 - -**缺点**： -- 需要管理多个 key -- 需要处理部分写入失败的情况 - -#### 方案 C：Stream 模式（使用 etcd 的 Watch） - -``` -mooncake-store/deletes/stream/{sequence_id} -``` - -**原理**： -- 每个批量写入一个 sequence_id -- Standby Watch 连续的 sequence_id -- 支持断点续传 - -**优点**： -- 支持顺序处理 -- 支持断点续传 -- 易于实现流式处理 - -**缺点**： -- 需要管理 sequence_id -- 需要处理 sequence_id 跳跃的情况 - -### 5. 实现细节 - -#### 5.1 BatchedDeleteEventManager - -```cpp -class BatchedDeleteEventManager { -public: - struct BatchConfig { - size_t max_batch_size = 1000; // 最大批量大小 - uint32_t max_batch_interval_ms = 1000; // 最大批量间隔（1秒） - }; - - // 添加 delete 事件到批量缓冲区 - void AddDeleteEvent(const std::string& key); - - // 强制刷新批量（立即写入） - void Flush(); - -private: - // 批量写入到 etcd - void FlushBatch(); - - // 压缩批量数据 - std::string CompressBatch(const std::vector& keys); - - // 解压缩批量数据 - std::vector DecompressBatch(const std::string& data); - - std::mutex mutex_; - std::vector pending_keys_; - std::chrono::steady_clock::time_point last_flush_time_; - BatchConfig config_; - std::thread flush_thread_; - std::atomic running_{false}; -}; -``` - -#### 5.2 批量写入逻辑 - -```cpp -void BatchedDeleteEventManager::FlushBatch() { - std::lock_guard lock(mutex_); - - if (pending_keys_.empty()) { - return; - } - - // 压缩数据 - std::string compressed_data = CompressBatch(pending_keys_); - - // 检查大小限制 - if (compressed_data.size() > kMaxEtcdValueSize) { - // 分片写入 - FlushBatchSharded(pending_keys_); - } else { - // 单个 key 写入 - FlushBatchSingle(compressed_data); - } - - pending_keys_.clear(); - last_flush_time_ = std::chrono::steady_clock::now(); -} -``` - -#### 5.3 Standby 端处理 - -```cpp -class DeleteEventWatcher { -public: - // Watch etcd delete events - void WatchDeleteEvents(); - - // 处理批量 delete 事件 - void HandleBatchDeleteEvent(const std::string& batch_data); - -private: - // 解压缩并应用 delete 事件 - void ApplyDeleteEvents(const std::vector& keys); -}; -``` - -## 方案评估 - -### 优点 - -1. **减少 etcd 压力** - - 从 130,000 次/秒 → 约 130 次/秒（批量 1000 条） - - 减少 1000 倍写入压力 - -2. **保持高可靠性** - - 仍然使用 etcd 的强一致性 - - Standby 可以通过 Watch 实时获取 - -3. **延迟可控** - - 批量间隔可配置（如 1 秒） - - 高吞吐时立即写入（大小阈值） - -4. **压缩减少存储** - - 使用压缩格式减少 etcd value 大小 - - 可以存储更多 delete 事件 - -### 缺点和挑战 - -1. **延迟问题** - - 批量写入会有延迟（最多 1 秒） - - 如果 Primary 在批量写入前崩溃，可能丢失部分 delete 事件 - -2. **数据丢失风险** - - 如果 Primary 在批量写入前崩溃，缓冲区中的 delete 事件会丢失 - - **解决方案**：使用持久化缓冲区（如 DragonflyDB）或定期 checkpoint - -3. **etcd 容量限制** - - etcd value 大小限制（1.5MB） - - 需要分片处理大批量 - -4. **压缩开销** - - 压缩/解压缩有 CPU 开销 - - 需要权衡压缩率和性能 - -5. **Standby 处理复杂度** - - 需要解压缩批量数据 - - 需要处理分片数据 - -### 与当前方案对比 - -| 特性 | 当前方案（不写入 etcd） | 新方案（批量写入 etcd） | -|------|------------------------|------------------------| -| **可靠性** | 中等（依赖租约同步） | 高（etcd 强一致性） | -| **延迟** | 0（实时） | 1 秒（批量延迟） | -| **etcd 压力** | 0 | 低（批量写入） | -| **数据丢失风险** | 低（Standby 自己决定） | 中等（批量缓冲区可能丢失） | -| **实现复杂度** | 低 | 中等 | -| **Standby 一致性** | 可能不一致（租约时间差） | 强一致（etcd 保证） | - -## 推荐方案 - -### 混合方案（推荐） - -**核心思想**： -1. **显式 Delete 事件**：立即写入 etcd（保持当前设计） -2. **驱逐 Delete 事件**：批量压缩写入 etcd（新方案） - -**实现策略**： -- 使用**混合策略**（大小阈值 + 时间窗口） - - 大小阈值：1000 条 - - 时间窗口：1 秒 -- 使用**简单列表格式**（中等数量 key） - - 如果 key 数量 > 1000，自动分片 -- 使用**分片 Key** 避免单个 value 过大 -- 添加**持久化缓冲区**（可选） - - 使用 DragonflyDB 作为缓冲区 - - 定期 checkpoint 到 etcd - -### 实施步骤 - -#### Phase 1：基础批量写入（低风险） - -1. 实现 `BatchedDeleteEventManager` -2. 使用简单列表格式 -3. 使用时间窗口批量（1 秒） -4. 单个 etcd key 写入 - -#### Phase 2：优化批量策略（中风险） - -1. 添加大小阈值 -2. 实现分片写入 -3. 添加压缩格式 - -#### Phase 3：持久化缓冲区（可选，高风险） - -1. 使用 DragonflyDB 作为缓冲区 -2. 定期 checkpoint 到 etcd -3. 故障恢复机制 - -## 总结 - -### 方案可行性：✅ **可行** - -**优点**： -- 大幅减少 etcd 压力（1000 倍减少） -- 保持高可靠性（etcd 强一致性） -- 延迟可控（1 秒内） - -**需要注意**： -- 批量延迟（最多 1 秒） -- 数据丢失风险（需要持久化缓冲区） -- etcd 容量限制（需要分片） - -### 建议 - -1. **先实现 Phase 1**（基础批量写入） - - 验证方案可行性 - - 评估性能影响 - -2. **根据实际效果决定是否继续** - - 如果效果良好，继续 Phase 2 - - 如果效果不佳，考虑其他方案 - -3. **关键指标**： - - etcd 写入 QPS - - Standby 同步延迟 - - 数据丢失率 - diff --git a/doc/zh/rfc-batched-delete-timing-issues.md b/doc/zh/rfc-batched-delete-timing-issues.md deleted file mode 100644 index d770f7f699..0000000000 --- a/doc/zh/rfc-batched-delete-timing-issues.md +++ /dev/null @@ -1,419 +0,0 @@ -# 批量写入 etcd 的时序问题分析 - -## 问题概述 - -批量写入 etcd 的方案可能存在以下时序问题： - -1. **事件顺序问题**：批量写入可能导致事件顺序混乱 -2. **竞态条件**：Standby 可能在不同时间看到不同批次的事件 -3. **重复删除问题**：同一个 key 可能出现在多个批次中 -4. **延迟导致的不一致**：批量延迟可能导致 Standby 看到过期数据 - -## 时序问题详细分析 - -### 问题 1：事件顺序混乱 - -#### 场景描述 - -``` -时间线： -T1: 驱逐 key1 → 加入 batch1 -T2: 驱逐 key2 → 加入 batch1 -T3: 显式删除 key1 → 立即写入 etcd (单个事件) -T4: batch1 写入 etcd (包含 key1, key2) -``` - -**问题**： -- Standby 在 T3 看到 key1 被删除（显式删除） -- Standby 在 T4 又看到 key1 被删除（批量删除） -- 或者 Standby 先看到 T4 的批量删除，后看到 T3 的显式删除 - -#### 影响 - -1. **重复删除**：Standby 可能尝试删除同一个 key 两次 - - 影响：性能开销，但通常可以容忍（幂等操作） - -2. **顺序混乱**：如果 key1 在 T3 被显式删除，但在 T4 的批量中又出现 - - 影响：Standby 可能看到"删除 → 存在 → 删除"的奇怪序列 - -### 问题 2：批量延迟导致的不一致 - -#### 场景描述 - -``` -时间线： -T1: 驱逐 key1 → 加入 batch1（未写入 etcd） -T2: Standby 读取 key1 → 看到 key1 存在（因为 batch1 还没写入） -T3: batch1 写入 etcd（包含 key1 的删除） -T4: Standby Watch 到 key1 被删除 -``` - -**问题**： -- T1-T3 期间，Standby 可能看到过期的 key1 -- 如果 Standby 在 T2 读取 key1，然后在 T4 看到删除，可能导致不一致 - -#### 影响 - -1. **短暂的不一致**：Standby 可能在短时间内看到 Primary 已经删除的 key - - 影响：可能导致 Standby 返回过期数据 - -2. **租约续约问题**：如果 Standby 在 T2 续约了 key1 的租约，但 key1 在 T1 已经被删除 - - 影响：Standby 可能续约了不存在的 key - -### 问题 3：批量边界导致的事件丢失 - -#### 场景描述 - -``` -时间线： -T1: 驱逐 key1 → 加入 batch1 -T2: batch1 达到阈值（1000条）→ 开始写入 etcd -T3: 驱逐 key2 → 加入 batch2（新批次） -T4: batch1 写入完成 -T5: Primary 崩溃 -``` - -**问题**： -- batch1 中的 key1 已经写入 etcd（Standby 能看到） -- batch2 中的 key2 还未写入 etcd（Standby 看不到） -- 如果 Primary 在 T5 崩溃，batch2 中的事件会丢失 - -#### 影响 - -1. **部分事件丢失**：Standby 可能只看到部分删除事件 - - 影响：Standby 和 Primary 的数据不一致 - -2. **恢复困难**：Primary 恢复后，无法知道哪些 key 应该被删除 - - 影响：需要重新同步或清理 - -### 问题 4：Watch 顺序问题 - -#### 场景描述 - -``` -时间线： -T1: batch1 写入 etcd (seq=100, keys=[key1, key2]) -T2: 显式删除 key3 → 立即写入 etcd (seq=101) -T3: batch2 写入 etcd (seq=102, keys=[key4, key5]) -``` - -**Standby Watch 顺序**： -- 如果 Standby 的 Watch 是顺序的，会按 seq=100, 101, 102 的顺序看到 -- 但如果 etcd 的 Watch 有延迟，可能看到不同的顺序 - -#### 影响 - -1. **事件顺序保证**：etcd 的 Watch 保证顺序，但批量写入可能打乱逻辑顺序 - - 影响：Standby 可能看到"批量删除 key1 → 显式删除 key3 → 批量删除 key2"的序列 - -2. **时间戳混乱**：批量中的 key 可能有不同的实际删除时间，但共享同一个时间戳 - - 影响：Standby 无法区分 key 的实际删除顺序 - -## 解决方案 - -### 方案 1：时间戳 + 序列号（推荐） - -#### 设计 - -每个 delete 事件包含： -- `timestamp`：实际删除时间（微秒精度） -- `sequence_id`：全局序列号（保证顺序） -- `batch_id`：批次 ID（用于去重） - -```json -{ - "batch_id": "2024-01-01T12:00:00.000Z", - "events": [ - { - "key": "key1", - "timestamp": 1704110400123456, // 实际删除时间 - "sequence_id": 1001, - "source": "eviction" - }, - { - "key": "key2", - "timestamp": 1704110400123457, - "sequence_id": 1002, - "source": "eviction" - } - ] -} -``` - -#### 优点 - -- 保持事件的实际顺序 -- 支持去重（通过 sequence_id） -- 支持时间戳排序 - -#### 缺点 - -- 需要维护全局序列号 -- 实现复杂度稍高 - -### 方案 2：去重机制 - -#### 设计 - -在 Standby 端维护一个"已删除 key"的集合，用于去重： - -```cpp -class DeleteEventProcessor { -private: - std::unordered_set deleted_keys_; - std::mutex mutex_; - -public: - void ProcessDeleteEvent(const std::string& key) { - std::lock_guard lock(mutex_); - - // 去重：如果已经删除过，跳过 - if (deleted_keys_.find(key) != deleted_keys_.end()) { - VLOG(1) << "Key " << key << " already deleted, skipping"; - return; - } - - // 执行删除 - DeleteKey(key); - deleted_keys_.insert(key); - - // 定期清理 deleted_keys_（避免内存泄漏） - if (deleted_keys_.size() > 100000) { - CleanupDeletedKeys(); - } - } -}; -``` - -#### 优点 - -- 简单易实现 -- 有效防止重复删除 -- 性能开销小 - -#### 缺点 - -- 需要维护内存中的集合 -- 需要定期清理（避免内存泄漏） - -### 方案 3：版本号机制 - -#### 设计 - -每个 delete 事件包含版本号，Standby 只处理版本号更高的删除事件： - -```json -{ - "batch_id": "2024-01-01T12:00:00.000Z", - "version": 100, // 全局版本号 - "events": [ - { - "key": "key1", - "key_version": 50, // key 的版本号 - "timestamp": 1704110400123456 - } - ] -} -``` - -#### 优点 - -- 支持版本比较 -- 可以检测过期事件 - -#### 缺点 - -- 需要维护版本号 -- 实现复杂度高 - -### 方案 4：分离显式删除和批量删除 - -#### 设计 - -使用不同的 etcd key 前缀区分显式删除和批量删除： - -``` -mooncake-store/deletes/explicit/{key_hash} # 显式删除 -mooncake-store/deletes/batch/{batch_id} # 批量删除 -``` - -Standby 处理逻辑： -1. 先处理显式删除（优先级高） -2. 再处理批量删除（去重） - -#### 优点 - -- 清晰区分两种删除类型 -- 可以设置不同的优先级 - -#### 缺点 - -- 需要维护两套逻辑 -- 可能增加 etcd key 数量 - -### 方案 5：事务保证原子性 - -#### 设计 - -使用 etcd 事务保证批量写入的原子性： - -```cpp -void BatchedDeleteEventManager::FlushBatch() { - // 构建事务 - etcd::Transaction txn; - - for (const auto& key : pending_keys_) { - std::string etcd_key = BuildDeleteKey(key); - txn.Put(etcd_key, SerializeDeleteEvent(key)); - } - - // 提交事务（原子性保证） - auto result = etcd_client_.Commit(txn); - if (!result.success) { - LOG(ERROR) << "Failed to commit batch delete events"; - // 重试或持久化到缓冲区 - } -} -``` - -#### 优点 - -- 保证批量写入的原子性 -- 要么全部成功，要么全部失败 - -#### 缺点 - -- etcd 事务有性能开销 -- 如果批量很大，事务可能失败 - -## 推荐方案：组合方案 - -### 核心设计 - -1. **时间戳 + 序列号**：每个事件包含实际删除时间和序列号 -2. **去重机制**：Standby 端维护已删除 key 集合 -3. **分离显式删除和批量删除**：使用不同的 etcd key 前缀 -4. **持久化缓冲区**：使用 DragonflyDB 作为缓冲区，避免数据丢失 - -### 实现示例 - -#### Primary 端 - -```cpp -class BatchedDeleteEventManager { -private: - uint64_t global_sequence_id_{0}; - std::mutex sequence_mutex_; - - struct DeleteEvent { - std::string key; - uint64_t timestamp; // 实际删除时间 - uint64_t sequence_id; // 全局序列号 - std::string source; // "explicit" or "eviction" - }; - - void FlushBatch() { - std::lock_guard lock(mutex_); - - if (pending_events_.empty()) { - return; - } - - // 分配序列号 - uint64_t batch_start_seq = GetNextSequenceId(pending_events_.size()); - - // 构建批量事件 - BatchDeleteEvent batch; - batch.batch_id = GenerateBatchId(); - batch.version = batch_start_seq; - - for (size_t i = 0; i < pending_events_.size(); ++i) { - auto& event = pending_events_[i]; - event.sequence_id = batch_start_seq + i; - batch.events.push_back(event); - } - - // 写入 etcd - WriteBatchToEtcd(batch); - - pending_events_.clear(); - } - - uint64_t GetNextSequenceId(size_t count) { - std::lock_guard lock(sequence_mutex_); - uint64_t start = global_sequence_id_; - global_sequence_id_ += count; - return start; - } -}; -``` - -#### Standby 端 - -```cpp -class DeleteEventProcessor { -private: - std::unordered_map deleted_keys_; // key -> max_sequence_id - std::mutex mutex_; - -public: - void ProcessBatchDeleteEvent(const BatchDeleteEvent& batch) { - std::lock_guard lock(mutex_); - - for (const auto& event : batch.events) { - // 去重：如果已经删除过，且序列号更小，跳过 - auto it = deleted_keys_.find(event.key); - if (it != deleted_keys_.end() && it->second >= event.sequence_id) { - VLOG(1) << "Key " << event.key - << " already deleted with sequence_id=" << it->second - << ", skipping sequence_id=" << event.sequence_id; - continue; - } - - // 执行删除 - DeleteKey(event.key); - deleted_keys_[event.key] = event.sequence_id; - } - - // 定期清理（保留最近 100000 个 key） - if (deleted_keys_.size() > 100000) { - CleanupOldKeys(); - } - } - - void ProcessExplicitDeleteEvent(const std::string& key, uint64_t sequence_id) { - std::lock_guard lock(mutex_); - - // 显式删除优先级更高，直接删除 - DeleteKey(key); - deleted_keys_[key] = sequence_id; - } -}; -``` - -## 时序问题总结 - -### 主要问题 - -1. **事件顺序混乱**：批量写入可能打乱事件的实际顺序 - - **解决方案**：使用时间戳 + 序列号 - -2. **重复删除**：同一个 key 可能出现在多个批次中 - - **解决方案**：Standby 端去重机制 - -3. **延迟不一致**：批量延迟可能导致 Standby 看到过期数据 - - **解决方案**：这是批量方案的固有特性，需要权衡 - -4. **数据丢失**：Primary 崩溃可能导致未写入的事件丢失 - - **解决方案**：持久化缓冲区（DragonflyDB） - -### 推荐方案 - -**组合方案**： -1. 时间戳 + 序列号（保证顺序） -2. Standby 端去重（防止重复删除） -3. 分离显式删除和批量删除（优先级区分） -4. 持久化缓冲区（避免数据丢失） - -这样可以最大程度地减少时序问题，同时保持批量写入的性能优势。 - diff --git a/doc/zh/rfc-delete-via-etcd-solution.md b/doc/zh/rfc-delete-via-etcd-solution.md deleted file mode 100644 index a22b8d9fb9..0000000000 --- a/doc/zh/rfc-delete-via-etcd-solution.md +++ /dev/null @@ -1,427 +0,0 @@ -# 基于 etcd 的 Delete 事件同步方案 - -## 方案概述 - -将 Delete 事件写入 etcd，利用 etcd 的强一致性和 watch 机制，确保所有 Standby Master 都能看到 Delete 事件，即使 Primary Master 崩溃。 - -## 方案设计 - -### 1. etcd Key 结构设计 - -``` -{etcd_prefix}/deletes/{cluster_id}/{key_hash} -``` - -示例： -``` -mooncake-store/deletes/mooncake_cluster/abc123def456 -``` - -**设计考虑**： -- 使用 `key_hash` 而不是原始 key，避免 etcd key 过长 -- 使用 `cluster_id` 支持多集群隔离 -- 使用统一的 `deletes` 前缀，便于批量管理 - -### 2. Delete 事件数据结构 - -```cpp -struct DeleteEvent { - std::string key; // 原始 key - uint64_t timestamp; // 删除时间戳 - ViewVersionId master_version; // Master view version（用于去重） - std::string master_address; // 执行删除的 Master 地址 -}; -``` - -序列化为 JSON 存储在 etcd value 中。 - -### 3. Primary Master：写入 Delete 事件 - -```cpp -auto MasterService::Remove(const std::string& key) - -> tl::expected { - // 1. 执行本地删除 - auto result = RemoveLocal(key); - if (!result) { - return result; - } - - // 2. 写入 Delete 事件到 etcd - if (enable_ha_) { - DeleteEvent event; - event.key = key; - event.timestamp = NowInMicroseconds(); - event.master_version = current_view_version_; - event.master_address = local_address_; - - std::string etcd_key = BuildDeleteKey(key); - std::string etcd_value = SerializeDeleteEvent(event); - - auto etcd_result = EtcdHelper::Put(etcd_key, etcd_value); - if (etcd_result != ErrorCode::OK) { - LOG(WARNING) << "Failed to write delete event to etcd: " - << etcd_result - << ", but local delete succeeded"; - // 继续执行，不阻塞删除操作 - } - } - - return {}; -} -``` - -### 4. Standby Master：Watch Delete 事件 - -```cpp -class DeleteEventWatcher { -public: - void StartWatching() { - watch_thread_ = std::thread([this]() { - WatchDeleteEvents(); - }); - } - -private: - void WatchDeleteEvents() { - std::string watch_prefix = etcd_prefix_ + "/deletes/" + cluster_id_ + "/"; - - // 使用 etcd watch 监听所有 delete 事件 - while (running_) { - auto watch_result = EtcdHelper::WatchPrefix(watch_prefix); - - for (const auto& event : watch_result.events) { - if (event.type == EventType::PUT) { - // 新的 Delete 事件 - ProcessDeleteEvent(event.key, event.value); - } else if (event.type == EventType::DELETE) { - // Delete 事件被清理（过期） - // 可以忽略 - } - } - } - } - - void ProcessDeleteEvent(const std::string& etcd_key, - const std::string& etcd_value) { - // 1. 解析 Delete 事件 - DeleteEvent event = DeserializeDeleteEvent(etcd_value); - - // 2. 检查是否已经处理过（去重） - if (processed_deletes_.count(event.key) > 0) { - return; // 已处理，跳过 - } - - // 3. 更新本地 metadata - if (hot_standby_service_) { - hot_standby_service_->ApplyDelete(event.key); - } - - // 4. 标记为已处理 - processed_deletes_.insert(event.key); - } -}; -``` - -### 5. 事件清理机制 - -为了避免 etcd 中积累大量 Delete 事件，需要定期清理： - -```cpp -class DeleteEventCleaner { -public: - void StartCleaning() { - cleaner_thread_ = std::thread([this]() { - while (running_) { - CleanOldDeleteEvents(); - std::this_thread::sleep_for( - std::chrono::minutes(cleanup_interval_minutes_)); - } - }); - } - -private: - void CleanOldDeleteEvents() { - std::string prefix = etcd_prefix_ + "/deletes/" + cluster_id_ + "/"; - - // 获取所有 Delete 事件 - auto all_events = EtcdHelper::List(prefix); - - auto now = NowInMicroseconds(); - for (const auto& event : all_events) { - DeleteEvent delete_event = DeserializeDeleteEvent(event.value); - - // 如果事件超过保留时间（如 1 小时），删除 - if (now - delete_event.timestamp > - kDeleteEventRetentionTimeUs) { - EtcdHelper::Delete(event.key); - } - } - } -}; -``` - ---- - -## 方案优势 - -### 1. ✅ 利用现有基础设施 - -- etcd 已经在使用（用于 Leader 选举） -- 不需要引入新的消息队列组件 -- 复用现有的 `EtcdHelper` 接口 - -### 2. ✅ 强一致性保证 - -- etcd 提供强一致性保证 -- 所有 Standby Master 都能看到相同的 Delete 事件 -- 即使 Primary 崩溃，事件仍然在 etcd 中 - -### 3. ✅ 实时同步 - -- etcd watch 机制可以实时推送 Delete 事件 -- Standby Master 可以立即响应 Delete 事件 -- 延迟通常在毫秒级 - -### 4. ✅ 持久化存储 - -- etcd 持久化存储，即使所有 Master 重启，事件仍然存在 -- 新启动的 Master 可以从 etcd 恢复历史 Delete 事件 - ---- - -## 潜在问题和解决方案 - -### 问题 1：etcd 性能和容量限制 - -**问题描述**： -- etcd 不适合存储大量数据 -- 大量 Delete 事件可能导致 etcd 性能下降 -- etcd 有存储容量限制（默认 2GB） - -**解决方案**： - -#### 方案 A：批量写入 + 定期清理 - -```cpp -// 批量收集 Delete 事件 -class DeleteEventBuffer { - std::vector buffer_; - std::mutex mutex_; - - void Flush() { - std::lock_guard lock(mutex_); - if (buffer_.empty()) return; - - // 批量写入 etcd（使用事务） - EtcdHelper::BatchPut(delete_events_); - buffer_.clear(); - } -}; -``` - -- 批量写入减少 etcd 压力 -- 定期清理旧事件，控制 etcd 存储量 - -#### 方案 B：只存储关键 Delete 事件 - -```cpp -// 只存储"高风险"的 Delete 事件 -bool ShouldStoreDeleteEvent(const std::string& key) { - // 只存储： - // 1. 最近活跃的 key（在 LRU 缓存中） - // 2. 有特殊标记的 key - // 3. 大对象的 key - return IsRecentlyActive(key) || HasSpecialFlag(key) || IsLargeObject(key); -} -``` - -- 只存储可能被重用的 key 的 Delete 事件 -- 普通 key 的 Delete 事件可以丢失（符合你的语义） - -#### 方案 C：使用 etcd 的 TTL 自动过期 - -```cpp -// 写入 Delete 事件时设置 TTL -EtcdHelper::PutWithTTL(etcd_key, etcd_value, - kDeleteEventTTLSeconds); // 如 60 秒 -``` - -- 利用 etcd 的 TTL 机制自动清理 -- 不需要额外的清理线程 - -### 问题 2：etcd Watch 延迟 - -**问题描述**： -- etcd watch 可能有延迟（网络、负载等） -- 在 watch 延迟期间，可能错过 Delete 事件 - -**解决方案**： - -#### 方案 A：Watch + 定期全量同步 - -```cpp -void SyncDeleteEvents() { - // 1. Watch 实时事件 - StartWatching(); - - // 2. 定期全量同步（作为兜底） - sync_thread_ = std::thread([this]() { - while (running_) { - FullSyncDeleteEvents(); - std::this_thread::sleep_for( - std::chrono::seconds(sync_interval_seconds_)); - } - }); -} - -void FullSyncDeleteEvents() { - // 获取 etcd 中所有 Delete 事件 - auto all_events = EtcdHelper::List(delete_prefix_); - - // 与本地 metadata 对比，补漏 - for (const auto& event : all_events) { - if (!IsDeletedLocally(event.key)) { - ProcessDeleteEvent(event.key, event.value); - } - } -} -``` - -#### 方案 B：使用 etcd 的 Revision 机制 - -```cpp -// 记录最后处理的 revision -int64_t last_processed_revision_ = 0; - -void WatchDeleteEvents() { - // 从上次的 revision 开始 watch - auto watch_result = EtcdHelper::WatchFromRevision( - delete_prefix_, last_processed_revision_); - - // 处理所有事件（包括历史事件） - for (const auto& event : watch_result.events) { - ProcessDeleteEvent(event); - last_processed_revision_ = event.revision; - } -} -``` - -### 问题 3：etcd 故障处理 - -**问题描述**： -- etcd 故障时，无法写入/读取 Delete 事件 -- 需要降级策略 - -**解决方案**： - -#### 方案 A：优雅降级 - -```cpp -auto MasterService::Remove(const std::string& key) - -> tl::expected { - // 1. 执行本地删除（必须成功） - auto result = RemoveLocal(key); - if (!result) { - return result; - } - - // 2. 尝试写入 etcd（可选） - if (enable_ha_ && etcd_available_) { - auto etcd_result = WriteDeleteEventToEtcd(key); - if (etcd_result != ErrorCode::OK) { - LOG(WARNING) << "etcd unavailable, delete event not synced"; - // 继续执行，不阻塞 - } - } - - return {}; -} -``` - -- etcd 故障时，Delete 操作仍然成功 -- 只是 Delete 事件可能丢失（符合你的语义） - -#### 方案 B：重试机制 - -```cpp -void WriteDeleteEventWithRetry(const std::string& key) { - int retries = 3; - while (retries > 0) { - auto result = EtcdHelper::Put(delete_key, delete_value); - if (result == ErrorCode::OK) { - return; - } - - retries--; - std::this_thread::sleep_for( - std::chrono::milliseconds(100 * (4 - retries))); - } - - LOG(WARNING) << "Failed to write delete event after retries"; -} -``` - ---- - -## 实现建议 - -### 阶段 1：基础实现 - -1. **实现 Delete 事件写入**： - - 在 `MasterService::Remove` 中写入 etcd - - 使用简单的 key-value 结构 - -2. **实现 Delete 事件 Watch**： - - Standby Master 启动 watch 线程 - - 处理 Delete 事件，更新本地 metadata - -3. **实现事件清理**： - - 使用 TTL 或定期清理 - -### 阶段 2：优化 - -1. **批量写入**：减少 etcd 压力 -2. **选择性存储**：只存储关键 Delete 事件 -3. **全量同步**：作为 watch 的兜底 - -### 阶段 3：生产就绪 - -1. **监控和告警**：监控 etcd 性能和容量 -2. **故障处理**：完善的降级策略 -3. **性能测试**：验证大量 Delete 事件的性能 - ---- - -## 与现有方案的对比 - -| 方案 | 优点 | 缺点 | -|------|------|------| -| **etcd Delete 事件** | ✅ 利用现有基础设施
✅ 强一致性
✅ 实时同步 | ⚠️ etcd 性能限制
⚠️ 需要清理机制 | -| **延迟物理删除** | ✅ 实现简单
✅ 不依赖外部组件 | ❌ 内存浪费 | -| **消息队列（EDQ/Kafka）** | ✅ 高性能
✅ 大容量 | ❌ 需要新组件
❌ 增加系统复杂度 | -| **Raft 协议** | ✅ 完全强一致 | ❌ 实现复杂
❌ 性能开销大 | - ---- - -## 总结 - -**将 Delete 事件写入 etcd 的方案是可行的**，但需要注意： - -1. **etcd 性能限制**： - - 需要批量写入和定期清理 - - 或者只存储关键 Delete 事件 - -2. **Watch 延迟**： - - 需要定期全量同步作为兜底 - - 或使用 revision 机制 - -3. **故障处理**： - - 需要优雅降级策略 - - etcd 故障时，Delete 操作仍然成功 - -**推荐实现方式**： -- **基础版本**：写入所有 Delete 事件 + TTL 自动清理 -- **优化版本**：只存储关键 Delete 事件 + 批量写入 + 定期全量同步 - -这个方案在**利用现有基础设施**和**解决 Delete 未同步问题**之间取得了很好的平衡。 - diff --git a/doc/zh/rfc-dragonflydb-as-consistency-store.md b/doc/zh/rfc-dragonflydb-as-consistency-store.md deleted file mode 100644 index 9e6bdaa4b0..0000000000 --- a/doc/zh/rfc-dragonflydb-as-consistency-store.md +++ /dev/null @@ -1,313 +0,0 @@ -# 使用 DragonflyDB 作为一致性中间存储组件的可行性分析 - -## 当前系统对 etcd 的使用场景 - -### 1. Leader Election（主从选举） -- **功能**：使用 etcd 的 Lease 机制和事务实现分布式锁 -- **关键操作**： - - `GrantLease()`：创建租约（TTL = 5秒） - - `CreateWithLease()`：使用事务创建 key（原子性保证） - - `KeepAlive()`：续租，保持 leader 身份 - - `WatchUntilDeleted()`：监听 leader key 删除，触发重新选举 - -### 2. Delete 事件同步 -- **功能**：将 Delete 事件写入 etcd，确保所有 Standby 都能看到 -- **关键操作**： - - `Put()`：写入 Delete 事件 - - `Watch()`：Standby 监听 Delete 事件 - - 需要强一致性保证 - -### 3. Metadata 存储（部分场景） -- **功能**：存储部分 metadata 信息 -- **关键操作**： - - `Get()` / `Put()`：读写 metadata - - `Update()`：带版本号的更新（使用事务） - -## DragonflyDB 特性分析 - -### 优势 -1. **高性能**：单机性能远超 Redis，适合高吞吐场景 -2. **Redis 协议兼容**：可以使用现有的 Redis 客户端库 -3. **内存数据库**：低延迟，适合实时同步场景 -4. **数据持久化**：支持快照和 AOF - -### 劣势和限制 -1. **分布式一致性协议支持不明确** - - 未明确支持 Raft/Paxos 等分布式一致性协议 - - 可能无法提供 etcd 级别的强一致性保证 - -2. **缺少关键特性** - - **Lease/TTL 机制**：Redis 有 `EXPIRE`，但可能不如 etcd 的 Lease 精确 - - **事务原子性**：Redis 有 `MULTI/EXEC`，但可能不如 etcd 的事务强大 - - **Watch 机制**：Redis 有 `PUBSUB` 和 `KEYSpace notifications`，但可能不如 etcd 的 Watch 可靠 - - **版本号/Revision**：Redis 没有内置的版本号机制 - -3. **集群模式** - - DragonflyDB 的集群模式可能使用主从复制或分片 - - 可能无法提供 etcd 的线性一致性（Linearizability） - -## 使用方案对比 - -### 方案 A：完全替代 etcd（不推荐） - -**优点**： -- 统一存储组件，简化架构 -- 高性能，低延迟 - -**缺点**： -- **Leader Election 风险**：Redis 的 `SET NX EX` 可能不如 etcd 的事务可靠 -- **一致性风险**：可能无法保证强一致性 -- **Watch 机制**：Redis 的 PUBSUB 可能丢失消息 -- **版本控制**：需要自己实现版本号机制 - -**实现示例**： -```cpp -// Leader Election（使用 Redis SET NX EX） -bool ElectLeader(const std::string& key, const std::string& value, int ttl) { - // Redis: SET key value NX EX ttl - // 问题：如果网络分区，可能出现多个 leader -} - -// Delete 事件同步（使用 Redis PUBSUB） -void PublishDeleteEvent(const std::string& key) { - // Redis: PUBLISH delete_channel delete_event_json - // 问题：如果 Standby 断开连接，可能丢失消息 -} -``` - -### 方案 B：混合方案（推荐） - -**架构**： -- **etcd**：继续用于 Leader Election（强一致性要求） -- **DragonflyDB**：用于 OpLog 存储和 Delete 事件同步（高性能要求） - -**优点**： -- 保留 etcd 的强一致性保证（Leader Election） -- 利用 DragonflyDB 的高性能（OpLog 和 Delete 事件） -- 各取所长 - -**缺点**： -- 需要维护两个存储组件 -- 架构稍复杂 - -**实现示例**： -```cpp -// Leader Election：继续使用 etcd -ErrorCode ElectLeader() { - return EtcdHelper::CreateWithLease(...); -} - -// OpLog 存储：使用 DragonflyDB -class DragonflyOpLogStore { - // 使用 Redis List 存储 OpLog - // LPUSH oplog:entries {seq_id, op_type, key, payload} - // LRANGE oplog:entries start end -}; - -// Delete 事件：使用 DragonflyDB Stream（Redis Stream） -void PublishDeleteEvent(const std::string& key) { - // Redis Stream: XADD delete_stream * key value - // Standby: XREAD BLOCK 0 STREAMS delete_stream $ -} -``` - -### 方案 C：DragonflyDB 作为 OpLog 持久化存储（推荐） - -**架构**： -- **etcd**：继续用于 Leader Election 和 Delete 事件（强一致性） -- **DragonflyDB**：仅用于 OpLog 的持久化存储和快速同步 - -**优点**： -- 最小化风险，只替换非关键路径 -- OpLog 可以容忍一定程度的丢失（有快照机制） -- 利用 DragonflyDB 的高性能加速 OpLog 同步 - -**实现示例**： -```cpp -class DragonflyOpLogStore { -public: - // 追加 OpLog 到 DragonflyDB - void AppendOpLog(const OpLogEntry& entry) { - // Redis List: LPUSH oplog:entries {json} - // 或 Redis Stream: XADD oplog_stream * {json} - } - - // Standby 从 DragonflyDB 拉取 OpLog - std::vector GetOpLogSince(uint64_t seq_id) { - // Redis Stream: XREAD BLOCK 0 STREAMS oplog_stream last_id - // 或 Redis List: LRANGE oplog:entries start end - } -}; -``` - -## 详细对比分析 - -### 1. Leader Election - -| 特性 | etcd | DragonflyDB (Redis) | 结论 | -|------|------|---------------------|------| -| 原子性 | 事务保证 | SET NX EX（可能不够强） | **etcd 更可靠** | -| Lease 机制 | 原生支持 | EXPIRE（可能不够精确） | **etcd 更可靠** | -| Watch 可靠性 | 强一致性保证 | PUBSUB 可能丢失 | **etcd 更可靠** | -| 性能 | 中等 | 高 | DragonflyDB 更快 | - -**建议**：Leader Election 继续使用 etcd - -### 2. Delete 事件同步 - -| 特性 | etcd | DragonflyDB (Redis Stream) | 结论 | -|------|------|---------------------------|------| -| 一致性 | 强一致性 | 最终一致性（可能） | **etcd 更可靠** | -| 持久化 | 持久化 | 可配置持久化 | 两者都支持 | -| Watch/Stream | Watch 机制 | Stream 机制 | 两者都支持 | -| 性能 | 中等 | 高 | **DragonflyDB 更快** | -| 消息丢失 | 不会丢失 | 可能丢失（如果未持久化） | **etcd 更可靠** | - -**建议**： -- **方案 1**：继续使用 etcd（如果强一致性要求高） -- **方案 2**：使用 DragonflyDB Stream + 持久化（如果性能要求高，可以容忍少量丢失） - -### 3. OpLog 存储 - -| 特性 | 当前（内存） | DragonflyDB | 结论 | -|------|------------|-------------|------| -| 持久化 | 无 | 支持 | **DragonflyDB 更好** | -| 容量 | 有限（100K 条） | 大容量 | **DragonflyDB 更好** | -| 性能 | 极高 | 高 | 当前方案更快 | -| 一致性 | 不适用 | 最终一致性可接受 | 两者都可 | - -**建议**：**可以使用 DragonflyDB**，因为： -- OpLog 可以容忍一定程度的丢失（有快照机制） -- 需要持久化以支持新 Standby 的初始同步 -- 性能要求相对较低（异步同步） - -## 推荐方案：混合架构 - -### 架构设计 - -``` -┌─────────────────────────────────────────────────────────┐ -│ Primary Master │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ etcd │ │ DragonflyDB │ │ OpLogManager │ │ -│ │ (Leader │ │ (OpLog Store)│ │ (Memory) │ │ -│ │ Election) │ │ │ │ │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -└─────────────────────────────────────────────────────────┘ - │ │ │ - │ │ │ - ▼ ▼ ▼ -┌─────────────────────────────────────────────────────────┐ -│ Standby Masters │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ etcd │ │ DragonflyDB │ │ OpLogApplier│ │ -│ │ (Watch │ │ (Pull OpLog) │ │ │ │ -│ │ Leader) │ │ │ │ │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -└─────────────────────────────────────────────────────────┘ -``` - -### 具体实现 - -#### 1. Leader Election：继续使用 etcd -```cpp -// 保持不变 -ErrorCode ElectLeader() { - return EtcdHelper::CreateWithLease(...); -} -``` - -#### 2. OpLog 持久化：使用 DragonflyDB -```cpp -class DragonflyOpLogStore { -public: - // 追加 OpLog（异步） - void AppendOpLog(const OpLogEntry& entry) { - // 使用 Redis Stream - std::string json = SerializeOpLogEntry(entry); - redis_->XAdd("oplog_stream", "*", {{"entry", json}}); - } - - // Standby 拉取 OpLog - std::vector GetOpLogSince(const std::string& last_id) { - // XREAD BLOCK 0 STREAMS oplog_stream last_id - auto messages = redis_->XRead({"oplog_stream"}, {last_id}, 1000); - // 解析并返回 - } -}; -``` - -#### 3. Delete 事件：可选方案 - -**选项 A：继续使用 etcd（推荐）** -- 保证强一致性 -- 代码改动小 - -**选项 B：使用 DragonflyDB Stream** -- 高性能 -- 需要处理消息丢失场景 - -## 实施建议 - -### Phase 1：OpLog 持久化到 DragonflyDB（低风险） - -1. **实现 DragonflyOpLogStore** - - 使用 Redis Stream 存储 OpLog - - 异步写入，不阻塞主流程 - - 支持 Standby 拉取 - -2. **修改 OpLogManager** - - 添加可选的持久化后端 - - 保持内存 buffer 不变（性能） - -3. **修改 HotStandbyService** - - 支持从 DragonflyDB 拉取 OpLog - - 支持断点续传 - -**优点**： -- 风险低，不影响现有功能 -- 可以逐步迁移 -- 支持新 Standby 的初始同步 - -### Phase 2：评估 Delete 事件迁移（可选） - -1. **实现 DragonflyDeleteEventStore** - - 使用 Redis Stream - - 添加持久化配置 - - 处理消息丢失场景 - -2. **对比测试** - - 性能对比 - - 一致性测试 - - 故障场景测试 - -3. **决定是否迁移** - - 如果性能提升明显且一致性可接受，则迁移 - - 否则继续使用 etcd - -## 总结 - -### 可以使用 DragonflyDB 的场景 - -1. **OpLog 持久化存储**（推荐） - - 优点：持久化、大容量、高性能 - - 风险：低（有快照机制兜底） - -2. **Delete 事件同步**（可选） - - 优点：高性能 - - 风险：中等（需要评估一致性要求） - -### 不建议使用 DragonflyDB 的场景 - -1. **Leader Election**（不推荐） - - 需要强一致性保证 - - etcd 的事务和 Lease 机制更可靠 - -### 推荐方案 - -**混合架构**： -- **etcd**：Leader Election + Delete 事件（强一致性） -- **DragonflyDB**：OpLog 持久化存储（高性能 + 持久化） - -这样既保证了关键路径的强一致性，又利用了 DragonflyDB 的高性能优势。 - diff --git a/doc/zh/rfc-oplog-cleanup-start-sequence-id.md b/doc/zh/rfc-oplog-cleanup-start-sequence-id.md deleted file mode 100644 index 74ba123b0d..0000000000 --- a/doc/zh/rfc-oplog-cleanup-start-sequence-id.md +++ /dev/null @@ -1,507 +0,0 @@ -# OpLog 清理时如何获取 start_sequence_id - -## 问题 - -使用 `DeleteRange` 清理 etcd 中某个 `sequence_id` 之前的所有 OpLog 时，需要确定 `start_sequence_id`（范围的起始点）。 - -## 方案选择 - -### 方案对比 - -| 方案 | 可靠性 | 实现复杂度 | 性能 | 推荐度 | -|------|--------|-----------|------|--------| -| 方案1：维护"已清理到"记录 | 低（Primary切换会丢失） | 中 | 高 | ❌ | -| 方案2：从快照记录获取 | 中 | 中 | 高 | ⚠️ | -| **方案3：从etcd查询最小sequence_id** | **高** | **中** | **中** | **✅** | -| 方案4：固定从1开始 | 高 | 低 | 低 | ❌ | - -### 推荐方案：方案3（从etcd查询最小sequence_id） - -**选择理由**： -1. **可靠性高**：信息存储在 etcd 中，Primary 切换不会丢失 -2. **自动适应**：自动获取实际存在的最小 sequence_id -3. **容错性好**：可以结合快照记录作为 fallback -4. **无需维护额外状态**：不需要"已清理到"的 key - -## 方案3详细设计 - -### 核心思路 - -1. **从 etcd 查询当前最小的 OpLog sequence_id** - - 使用 `Get` with `WithPrefix` + `WithLimit(1)` + `WithSort` - - 获取第一个（最小的）OpLog key - -2. **Fallback 机制** - - 如果查询不到 OpLog，使用快照记录作为 fallback - - 如果快照记录也没有，使用保守策略（从 1 开始） - -3. **执行 DeleteRange** - - 从查询到的最小 sequence_id 开始删除 - - 到目标 sequence_id（不包含）结束 - -### 架构设计 - -``` -┌─────────────────────────────────────────────────────────┐ -│ CleanupOpLogBefore(target_sequence_id) │ -└─────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 1. GetMinSequenceId() │ -│ ┌──────────────────────────────────────┐ │ -│ │ GetFirstKeyWithPrefix(prefix) │ │ -│ │ - WithPrefix │ │ -│ │ - WithLimit(1) │ │ -│ │ - WithSort(SortByKey, SortAscend) │ │ -│ └──────────────────────────────────────┘ │ -│ │ │ -│ ├─ 成功 → 解析 sequence_id │ -│ │ │ -│ └─ 失败 → Fallback │ -│ │ │ -│ ├─ GetLastSnapshotSequenceId() │ -│ │ │ -│ └─ 都没有 → 使用 1（保守策略） │ -└─────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 2. DeleteRange(start_seq_id, target_sequence_id) │ -│ - start_key = BuildOpLogKey(start_seq_id) │ -│ - end_key = BuildOpLogKey(target_sequence_id) │ -│ - 执行 DeleteRange │ -└─────────────────────────────────────────────────────────┘ -``` - -## 实现细节 - -### 1. etcd Wrapper：GetFirstKeyWithPrefix - -**在 `etcd_wrapper.go` 中添加**： - -```go -//export EtcdStoreGetFirstKeyWithPrefixWrapper -func EtcdStoreGetFirstKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, - firstKey **C.char, firstKeySize *C.int, - firstValue **C.char, firstValueSize *C.int, - errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - - prefixStr := C.GoStringN(prefix, prefixSize) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - // 使用 Get with prefix，Limit=1，Sort=ASC 获取第一个 key - resp, err := storeClient.Get(ctx, prefixStr, - clientv3.WithPrefix(), - clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), - clientv3.WithLimit(1)) - if err != nil { - *errMsg = C.CString(err.Error()) - return -1 - } - - if len(resp.Kvs) == 0 { - // 没有找到，返回 -2 表示不存在 - *errMsg = C.CString("no key found with prefix") - return -2 - } - - // 返回第一个 key 和 value - kv := resp.Kvs[0] - *firstKey = C.CString(string(kv.Key)) - *firstKeySize = C.int(len(kv.Key)) - *firstValue = C.CString(string(kv.Value)) - *firstValueSize = C.int(len(kv.Value)) - - return 0 -} -``` - -### 2. C++ EtcdHelper：GetFirstKeyWithPrefix - -**在 `etcd_helper.h` 中添加**： - -```cpp -/** - * @brief Get the first key with a given prefix (sorted by key, ascending) - * @param prefix Key prefix - * @param prefix_size Size of prefix - * @param first_key Output: first key found - * @param first_value Output: value of first key - * @return ErrorCode::OK on success, ErrorCode::ETCD_KEY_NOT_EXIST if not found - */ -static ErrorCode GetFirstKeyWithPrefix(const char* prefix, size_t prefix_size, - std::string& first_key, std::string& first_value); -``` - -**在 `etcd_helper.cpp` 中实现**： - -```cpp -ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, size_t prefix_size, - std::string& first_key, std::string& first_value) { - char* err_msg = nullptr; - char* key_ptr = nullptr; - int key_size = 0; - char* value_ptr = nullptr; - int value_size = 0; - - int ret = EtcdStoreGetFirstKeyWithPrefixWrapper( - (char*)prefix, (int)prefix_size, - &key_ptr, &key_size, - &value_ptr, &value_size, - &err_msg); - - if (ret == -2) { - // 没有找到 - free(err_msg); - return ErrorCode::ETCD_KEY_NOT_EXIST; - } - - if (ret != 0) { - LOG(ERROR) << "Failed to get first key with prefix: " << err_msg; - free(err_msg); - return ErrorCode::ETCD_OPERATION_ERROR; - } - - first_key = std::string(key_ptr, key_size); - first_value = std::string(value_ptr, value_size); - - free(key_ptr); - free(value_ptr); - free(err_msg); - - return ErrorCode::OK; -} -``` - -### 3. EtcdOpLogStore：GetMinSequenceId - -**实现**： - -```cpp -uint64_t EtcdOpLogStore::GetMinSequenceId() const { - // 构建 OpLog 的 prefix - std::string prefix = etcd_prefix_ + "/" + cluster_id_ + "/"; - - // 查询第一个 OpLog key（最小的 sequence_id） - std::string first_key, first_value; - auto err = EtcdHelper::GetFirstKeyWithPrefix( - prefix.c_str(), prefix.size(), - first_key, first_value); - - if (err == ErrorCode::OK) { - // 成功获取，从 key 中提取 sequence_id - uint64_t min_seq_id = ExtractSequenceIdFromKey(first_key); - if (min_seq_id > 0) { - LOG(INFO) << "Found min sequence_id in etcd: " << min_seq_id; - return min_seq_id; - } - } - - // Fallback：尝试从快照记录获取 - uint64_t last_snapshot_seq_id = GetLastSnapshotSequenceId(); - if (last_snapshot_seq_id > 0) { - LOG(INFO) << "Using last snapshot sequence_id as fallback: " - << last_snapshot_seq_id; - return last_snapshot_seq_id; - } - - // 保守策略：从 1 开始 - // 注意：如果所有 OpLog 都被清理了，DeleteRange 会安全处理不存在的 key - LOG(INFO) << "No OpLog or snapshot found, using conservative start: 1"; - return 1; -} - -uint64_t EtcdOpLogStore::ExtractSequenceIdFromKey(const std::string& key) const { - // key 格式：mooncake-store/oplog/{cluster_id}/{sequence_id} - // 例如：mooncake-store/oplog/mooncake_cluster/12345 - - size_t last_slash = key.find_last_of('/'); - if (last_slash == std::string::npos) { - LOG(ERROR) << "Invalid OpLog key format: " << key; - return 0; - } - - std::string seq_id_str = key.substr(last_slash + 1); - try { - uint64_t sequence_id = std::stoull(seq_id_str); - return sequence_id; - } catch (const std::exception& e) { - LOG(ERROR) << "Failed to parse sequence_id from key: " << key - << ", error: " << e.what(); - return 0; - } -} -``` - -### 4. EtcdOpLogStore：CleanupOpLogBefore - -**实现**： - -```cpp -bool EtcdOpLogStore::CleanupOpLogBefore(uint64_t target_sequence_id) { - if (target_sequence_id <= 1) { - LOG(INFO) << "No OpLog to cleanup: target_sequence_id=" << target_sequence_id; - return true; // 没有需要清理的 - } - - // 1. 从 etcd 查询最小的 sequence_id - uint64_t min_seq_id = GetMinSequenceId(); - - // 2. 如果 min_seq_id >= target_sequence_id，无需清理 - if (min_seq_id >= target_sequence_id) { - LOG(INFO) << "No OpLog to cleanup: min_seq_id=" << min_seq_id - << " >= target_sequence_id=" << target_sequence_id; - return true; - } - - // 3. 执行 DeleteRange - std::string start_key = BuildOpLogKey(min_seq_id); - std::string end_key = BuildOpLogKey(target_sequence_id); - - LOG(INFO) << "Cleaning up OpLog from " << min_seq_id - << " to " << target_sequence_id; - - int64_t deleted_count = 0; - auto err = EtcdHelper::DeleteRange( - start_key.c_str(), start_key.size(), - end_key.c_str(), end_key.size(), - deleted_count); - - if (err != ErrorCode::OK) { - LOG(ERROR) << "Failed to cleanup OpLog from " << min_seq_id - << " to " << target_sequence_id; - return false; - } - - LOG(INFO) << "Successfully cleaned up " << deleted_count - << " OpLog entries from " << min_seq_id - << " to " << target_sequence_id; - return true; -} -``` - -### 5. EtcdHelper：DeleteRange - -**在 `etcd_helper.h` 中添加**： - -```cpp -/** - * @brief Delete a range of keys - * @param start_key Start key (inclusive) - * @param start_key_size Size of start_key - * @param end_key End key (exclusive) - * @param end_key_size Size of end_key - * @param deleted_count Output: number of keys deleted - * @return ErrorCode::OK on success - */ -static ErrorCode DeleteRange(const char* start_key, size_t start_key_size, - const char* end_key, size_t end_key_size, - int64_t& deleted_count); -``` - -**在 `etcd_wrapper.go` 中添加**： - -```go -//export EtcdStoreDeleteRangeWrapper -func EtcdStoreDeleteRangeWrapper(startKey *C.char, startKeySize C.int, - endKey *C.char, endKeySize C.int, - deletedCount *C.int64, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - - start := C.GoStringN(startKey, startKeySize) - end := C.GoStringN(endKey, endKeySize) - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - // 使用 WithRange 删除指定范围内的 key - resp, err := storeClient.Delete(ctx, start, clientv3.WithRange(end)) - if err != nil { - *errMsg = C.CString(err.Error()) - return -1 - } - - *deletedCount = C.int64(resp.Deleted) - return 0 -} -``` - -**在 `etcd_helper.cpp` 中实现**： - -```cpp -ErrorCode EtcdHelper::DeleteRange(const char* start_key, size_t start_key_size, - const char* end_key, size_t end_key_size, - int64_t& deleted_count) { - char* err_msg = nullptr; - int64_t deleted = 0; - int ret = EtcdStoreDeleteRangeWrapper( - (char*)start_key, (int)start_key_size, - (char*)end_key, (int)end_key_size, - &deleted, &err_msg); - - if (ret != 0) { - LOG(ERROR) << "Failed to delete range: " << err_msg; - free(err_msg); - return ErrorCode::ETCD_OPERATION_ERROR; - } - - deleted_count = deleted; - free(err_msg); - return ErrorCode::OK; -} -``` - -## 使用场景示例 - -### 场景 1：正常清理 - -``` -当前状态： -- etcd 中 OpLog: sequence_id = 1000, 1001, 1002, ..., 5000 -- 快照时 sequence_id = 5000 -- 需要清理 sequence_id < 5000 的 OpLog - -执行流程： -1. GetMinSequenceId() → 返回 1000 -2. DeleteRange(1000, 5000) → 删除 1000-4999 -3. 结果：etcd 中只剩下 sequence_id >= 5000 的 OpLog -``` - -### 场景 2：所有 OpLog 都被清理了 - -``` -当前状态： -- etcd 中没有 OpLog（都被清理了） -- 快照时 sequence_id = 10000 -- 需要清理 sequence_id < 10000 的 OpLog - -执行流程： -1. GetMinSequenceId() → 查询不到 OpLog -2. Fallback 到快照记录 → 返回 10000 -3. DeleteRange(10000, 10000) → 无需删除（范围为空） -4. 结果：安全处理，不会出错 -``` - -### 场景 3：Primary 切换后清理 - -``` -场景： -- 原 Primary 清理了 sequence_id < 5000 的 OpLog -- 原 Primary 崩溃，Standby 提升为新的 Primary -- 新 Primary 需要清理 sequence_id < 10000 的 OpLog - -执行流程： -1. GetMinSequenceId() → 从 etcd 查询，返回 5000（实际存在的最小值） -2. DeleteRange(5000, 10000) → 删除 5000-9999 -3. 结果：正确清理，不会重复删除已清理的 key -``` - -## 性能考虑 - -### 查询性能 - -- **GetFirstKeyWithPrefix**：使用 `WithLimit(1)`，只获取第一个 key -- **性能开销**：O(log n)，n 为 OpLog key 数量 -- **频率**：只在清理时执行（10 分钟一次），开销可接受 - -### 删除性能 - -- **DeleteRange**：etcd 原生支持，性能高效 -- **批量删除**：一次操作删除整个范围 -- **如果范围很大**：可以考虑分批删除（但通常不需要） - -## 容错机制 - -### 1. 查询失败处理 - -```cpp -if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { - // 没有 OpLog，使用 fallback - return GetLastSnapshotSequenceId(); -} -``` - -### 2. 解析失败处理 - -```cpp -try { - uint64_t sequence_id = std::stoull(seq_id_str); - return sequence_id; -} catch (const std::exception& e) { - // 解析失败，使用 fallback - return GetLastSnapshotSequenceId(); -} -``` - -### 3. DeleteRange 失败处理 - -```cpp -if (err != ErrorCode::OK) { - LOG(ERROR) << "Failed to cleanup OpLog"; - // 可以重试，或者记录错误，下次再试 - return false; -} -``` - -## 与快照集成 - -### 快照时清理 - -```cpp -class SnapshotManager { -public: - MetadataSnapshot CreateSnapshot() { - MetadataSnapshot snapshot; - - // 1. 导出 metadata - snapshot.metadata = ExportMetadata(); - - // 2. 记录当前的 OpLog sequence_id - snapshot.last_oplog_sequence_id = oplog_manager_->GetLastSequenceId(); - - // 3. 将快照信息写入 etcd - std::string snapshot_id = GenerateSnapshotId(); - etcd_oplog_store_->RecordSnapshotSequenceId( - snapshot_id, snapshot.last_oplog_sequence_id); - - // 4. 清理旧的 OpLog（使用方案3） - etcd_oplog_store_->CleanupOpLogBefore( - snapshot.last_oplog_sequence_id); - - return snapshot; - } -}; -``` - -## 总结 - -### 方案3的优势 - -1. **可靠性高**：信息存储在 etcd 中，Primary 切换不会丢失 -2. **自动适应**：自动获取实际存在的最小 sequence_id -3. **容错性好**：结合快照记录作为 fallback -4. **无需维护额外状态**：不需要"已清理到"的 key -5. **性能可接受**：查询只在清理时执行，频率低 - -### 关键实现点 - -1. **GetFirstKeyWithPrefix**：使用 etcd 的 `WithPrefix` + `WithLimit(1)` + `WithSort` -2. **ExtractSequenceIdFromKey**：从 key 中解析 sequence_id -3. **Fallback 机制**：快照记录 → 保守策略（从1开始） -4. **DeleteRange**：使用 etcd 的 `WithRange` 删除范围 - -### 注意事项 - -1. **Key 格式**：必须固定格式，便于解析 sequence_id -2. **错误处理**：完善的 fallback 机制 -3. **日志记录**：记录清理过程，便于排查问题 - diff --git a/doc/zh/rfc-oplog-hot-standby-complete.md b/doc/zh/rfc-oplog-hot-standby-complete.md deleted file mode 100644 index 6a743722dd..0000000000 --- a/doc/zh/rfc-oplog-hot-standby-complete.md +++ /dev/null @@ -1,364 +0,0 @@ -# 基于 etcd 的 OpLog 主备同步完整方案 RFC - -## 1. 方案背景 - -### 1.1 当前系统架构 - -Mooncake Store 是一个高性能的分布式 KV 缓存存储引擎，专为 LLM 推理场景设计。系统采用 Master-Client 架构： - -- **Master Service**：负责管理对象元数据（metadata）、空间分配、节点管理等 -- **Client**：作为存储服务器提供内存段，同时作为客户端处理应用请求 - -### 1.2 高可用性需求 - -当前系统支持两种部署模式： - -1. **默认模式**：单 Master 节点，部署简单但存在单点故障风险 -2. **高可用模式（不稳定）**：多 Master 节点通过 etcd 进行 Leader 选举 - -**问题**： -- 高可用模式虽然实现了 Leader 选举，但 Standby Master 在等待期间不执行任何操作 -- 没有实现数据同步机制，Standby 提升为 Primary 时 metadata 可能不完整 -- 缺乏可靠的主备数据同步方案 - -### 1.3 业务场景 - -在 LLM 推理场景中，Master Service 需要： -- **高可用性**：Master 故障时能够快速切换，最小化服务中断时间 -- **数据一致性**：Standby 必须与 Primary 保持数据一致 -- **快速恢复**：故障恢复后能够快速恢复服务，无需长时间的数据重建 - -### 1.4 现有方案的问题 - -1. **无数据同步**：Standby Master 在等待选举期间不执行任何数据同步操作 -2. **元数据丢失风险**：Primary 故障后，Standby 提升时 metadata 可能不完整 -3. **恢复时间长**：需要重新从 Client 节点收集 metadata，恢复时间长 -4. **数据不一致**：无法保证 Standby 与 Primary 的数据一致性 - -## 2. Goals（目标） - -### 2.1 主要目标 - -1. **实现可靠的主备数据同步** - - Primary Master 的所有 metadata 变更操作同步到 Standby Master - - 保证 Standby 与 Primary 的数据一致性 - -2. **快速故障恢复** - - Primary 故障后，Standby 能够快速提升为 Primary - - 提升时 metadata 完整，无需长时间重建 - -3. **最小化 OpLog 大小** - - 只记录关键的状态变更操作（PUT、DELETE） - - 不记录租约续约等高频但非关键操作 - -4. **与现有系统集成** - - 与现有的快照机制集成 - - 与现有的 Leader 选举机制集成 - - 不影响现有功能的正常运行 - -### 2.2 非功能性目标 - -1. **性能**：OpLog 同步不应显著影响 Primary 的性能 -2. **可靠性**：利用 etcd 的强一致性保证数据可靠性 -3. **可扩展性**：支持多个 Standby Master -4. **可维护性**：实现简单，易于理解和维护 - -## 3. Proposal（提案） - -### 3.1 核心设计思路 - -**使用 etcd 作为中间可靠性组件，实现 OpLog 主备同步**： - -1. **OpLog 机制**：Primary Master 记录所有状态变更操作到 OpLog -2. **etcd 存储**：OpLog 写入 etcd，利用 etcd 的强一致性和持久化能力 -3. **Watch 机制**：Standby Master 通过 etcd Watch 机制实时接收 OpLog -4. **顺序保证**：通过全局 sequence_id 和 key 级别的 key_sequence_id 保证操作顺序 - -### 3.2 架构设计 - -#### 3.2.1 整体架构 - -整体架构图展示了 Primary Master、etcd Cluster 和 Standby Master 之间的交互关系： - -![PlantUML Diagram](https://uml.planttext.com/plantuml/png/XLL1QnD15Bu7yX_6zgA149KYmOEqb0H5YyKSF1GfazrficHtPfsTBSGGi62Yr8g1Hb4R2R4jzD9O9OYchVwPx2QU_0lEx6oQtMwgUmXllldUUr_UjpCxRp58cMteW9WwAIIBX2KvXDLyEGcfKjGOKfXDKJnsYHMHWO2fGmt7OrP9moQaq01vg9GAbDXONIGweM0swpr1Ya8Cas24MOwLTGGeBmbnGKT1ZehMeAspBE4ixGa2rwx7O_6OoOl30W8porGp82s39MWnH6V0R2QTdSkcGIKU0_mvwm1M92E7wBgce4S0MY24HFZtpNkai0GnRqCzUX28i3DCKJr2ZX4gouSXcI5_Gur1CdahL1lS1AFkaNFwnjr-DJXjoPGGGMI4g_CSf_xUgUrBOZnM3Kq9SJ9Or6r_Hjo6kSoDyOnKo60UMWYi29edNGJdQ-Ia-vD9Pwzcqvd_JZfdcoAmY1pYP1b9kqsOtoDeqWITxj13o9IYxv0VJoSkcAQk-KG_ZYf738fnJ4mC8K4F9t_4isCYKrZH-Eni7gISZPPx-4dI0_k2xYlbN2yQkoQOuor1ytMAaltci7aGv8tt12-aahFTdPxxzWWOFknRUUwL4OdUYt7-tV70QIe7_PU3us-Y52QCdrUjK6I0h4LEHY8nsjWQzNOJYJyd7mIG1CDcsttH01PwR2Eie5LD3U4bL5wDxXtttCqzfrvp3jyDJxQT-bTdgy_bOHM8_b4T0LkdI71tdxhj_T-TljD_BH5dxzcmKH_y-7A6kDzh71dzUkwsskx7pd2d-wz-aGiaaP1dDj1rsMOPh5w-8bSFa47MqNYLuNbC8rYiB-uY3wCeVXUL-TNmSzJj11gal0iwLL7ayURJgwOgWLbMBwRfa26BoNtfyCBodR2KURxWNm4H_WK0) - -**架构说明**： -- **Primary Master**：负责处理客户端请求，记录 OpLog 并写入 etcd -- **etcd Cluster**：作为中间存储，提供强一致性和 Watch 机制 -- **Standby Master**：通过 Watch etcd 实时接收 OpLog，并应用到本地 metadata store - -#### 3.2.2 数据流图 - -数据流图展示了从 Client 请求到 Standby 同步的完整流程： - -![PlantUML Diagram](https://uml.planttext.com/plantuml/png/VLH1QnD15Bulx7zuraiAJNfVI6c8wKLZGcBfHGYJtN6pP3AxpaugGJnu4xHOi0Y284K4BxLUl1JyDtPZ_uLlPdSdiqamX_3oydtptllUDtEOIYBaVCOWJbWSrWCYIVqPYr-upZqveJCA2ICHTvrq6l6423A3CV6deOZdF6Z7B1Pm_qX_R4XAdyyfzscNfYa9QOj58GUVSac5wxWEyINosYp2ZEiWHKP-b10kOQSleXaH2-YI5C4xG58eKcl0Nl8e3hk4u_4vhAVwxuPY3TUHVg2nGwn9DLAbz2_NKUEEIKg1OcvRXHCY_KbHeOYtmLf9WjFai29UWmqbuS6uCbYHKeeqcz0_VZBgF7u0sOUpFxy_PxzUBx-_XMPJ_Pih1VM3KWiF-dFPuK5jIXTxq6WqThMeqIcHTALNgINoId4yrHr5Ob5j3_04cxnIiOogzEN5b-pDkLdmA0gUyYA79usiVFK4exa79oAILAjMmwb4fRor6XCgkbgFfoI2VUtJ_PTOwPL5pFUdlg5UBJUS-pxQ47TDrz1MXSgCsnXMOwknx8LK9hU8Aq7DEf2MRtHxARC_ROlIDxVdxxAhRnLRvDCUbBxqyW0wfyeijUpZJz0gs_eQ2nU1eXT-rTPW2qtfgBriRiSukmWgxFQ4-jDXeK9F15JK59T9kBkykRrvdrrzNLx-S1t0ZyKlvlFWEC7BY2-69EfIsivM3DE3kJGgMufJjnincYg4fQjXKeONFc_gxkBJt-lhZQRCMOEOCVNUjNWmeFWIBcgx6-3ScmDAydVcA1OFgS4PHveZDGYKmX5D_rDPbylHs38FBDNjdMzpaDcJbJERTvr3F0rVV1N-0m00) - -**流程说明**： -1. Client 发送 `PutEnd` 请求到 Primary Master -2. Primary Master 通过 `OpLogManager` 记录操作，生成 sequence_id -3. `EtcdOpLogStore` 将 OpLog 写入 etcd -4. etcd 通过 Watch 机制通知 Standby Master -5. `OpLogWatcher` 接收事件并传递给 `OpLogApplier` -6. `OpLogApplier` 检查顺序并应用到 Standby 的 metadata store - -#### 3.2.3 故障切换流程 - -故障切换流程图展示了从 Primary 故障到 Standby 提升为 Primary 的完整过程： - -![PlantUML Diagram](https://uml.planttext.com/plantuml/png/dPNFQnD15CVl2_i_FEkbFRJuynA8j6X4iCL24IzUfhlFTEbcTvsTXFQMWlqHRMilbL8Y9IhMWwq8Ah6A_MUoa-I_S6PszbSiYERqvittEtdlyuRPwP0Hoker5_p0zQkJJuZZ-Wsaao4-hQDdeMbSOajOGmXSudYc4IuxNa0egS4YiPQhrAzxzctVzIbSlgj-UKboo1o68QdYZEjKFR3GOqXDmpI4Y3cM4n2FmTWyTMg4hi8S2SNs690GTCeqRCB88WaHa5dsY6-14SzUBFXqQaGO2nQGDXmB5-g1349VEzBbYEcUp_HfsgZaMNP4_Y2OzQkF2BEMT2awlaWs4mIkesKwbb3APU0dRwDkTt2-D-Xi3m--yTElK2xBlOJHv2r5eWJt4GD1jO4mYuAFQSYqtCuQAiKrI86D5CQZauEe_M72D8Z5d0PXM6W-Y-KfMPyb2PKcg_6yFGyZYwLTDw-z1LFAHGTPIt6rYb3MJdgIoaEb8UvGM31hWYKLh2fPnMEqM6gAMGUALDBWmq1SCt5L6P7NJVfi_DEPowKzv79v6Bbq7h4QSJ99lhy-F6oFZdT5ir13XSfAu52qOJmMJrmyPJtVE-WY4sA5w3-6x0V_FjpOymza58BaBFvoBzhPx7NFKYWnZMALQOdprA_v30jLfWVdwaiDqTRhwFX5jFqgnldOuztr_jx6u7oJju-WfkTTyCRqAovQBCPQ-BVu4KfdaFoF7e1oeLteFNRaYyiDBdtuV1kBb-Ql5yaJ840-rvaMuEeKH6jjVl8c8zpUYPvtvDwrAHYkxKIx6xpLvErMhdk0wyBtwJku4bBv2lGFdudbu7E7xsxrphQ6Fmu6f-_wnqVzi_TIVMCAUEjOF52zRfD_x4Idstp_Y_2aHqACMMflYfD_DiKG4aR3PglN_IKOUZR87cGlqs8XlaFok_0R) - - - - -``` -**流程说明**： -1. **正常运行**：Primary 保持 Lease，Standby 通过 Watch 持续同步 OpLog -2. **Primary 故障**：Primary 的 Lease 过期，etcd 通知 Standby -3. **Standby 提升**：停止 Standby 服务，初始化 Lease，清理过期 metadata，开始 Leader 选举 -``` - -### 3.3 核心组件设计 - -#### 3.3.1 OpLogManager（Primary 端） - -**职责**： - -- 记录所有状态变更操作（PUT_END、PUT_REVOKE、REMOVE） -- 生成全局 sequence_id 和 key 级别的 key_sequence_id -- 维护内存缓冲区（用于快速查询） - -**关键方法**： -```cpp -class OpLogManager { - uint64_t Append(OpType type, const std::string& key, - const std::string& payload = ""); - std::vector GetEntriesSince(uint64_t since_seq_id, - size_t limit = 1000) const; - uint64_t GetLastSequenceId() const; -}; -``` - -#### 3.3.2 EtcdOpLogStore（Primary 端） - -**职责**： -- 将 OpLog 写入 etcd -- 更新最新的 sequence_id -- 记录快照对应的 sequence_id -- 清理旧的 OpLog - -**etcd Key 设计**： -- OpLog Entry: `mooncake-store/oplog/{cluster_id}/{sequence_id}` -- Latest Sequence ID: `mooncake-store/oplog/{cluster_id}/latest` -- Snapshot Sequence ID: `mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` - -#### 3.3.3 OpLogWatcher（Standby 端） - -**职责**： -- Watch etcd 的 OpLog 变化 -- 读取历史 OpLog（用于初始同步） -- 处理 Watch 事件并传递给 OpLogApplier - -**关键方法**： -```cpp -class OpLogWatcher { - void Start(); - void Stop(); - bool ReadOpLogSince(uint64_t start_seq_id, - std::vector& entries); -}; -``` - -#### 3.3.4 OpLogApplier（Standby 端） - -**职责**： -- 应用 OpLog Entry 到本地 metadata store -- 检查全局和 key 级别的顺序 -- 处理序列号不连续和乱序情况 -- 定期清理 key_sequence_map_（内存优化） - -**关键方法**： -```cpp -class OpLogApplier { - bool ApplyOpLogEntry(const OpLogEntry& entry); - bool CheckSequenceOrder(const OpLogEntry& entry); - void CleanupStaleKeySequences(); -}; -``` - -#### 3.3.5 HotStandbyService（Standby 端） - -**职责**： -- 管理 Standby 模式的生命周期 -- 协调 OpLogWatcher 和 OpLogApplier -- 处理 Standby 提升为 Primary 的逻辑 - -**关键方法**： -```cpp -class HotStandbyService { - void StartStandby(); - void Stop(); - void Promote(); -}; -``` - -### 3.4 OpLog Entry 数据结构 - -```cpp -struct OpLogEntry { - uint64_t sequence_id{0}; // 全局递增序列号 - uint64_t timestamp_ms{0}; // 时间戳（毫秒） - OpType op_type{OpType::PUT_END}; // PUT_END, PUT_REVOKE, REMOVE - std::string object_key; // 对象 key - std::string payload; // 可选负载（用于 PUT_END 时携带 replica 信息） - uint32_t checksum{0}; // 校验和 - uint32_t prefix_hash{0}; // key 前缀哈希 - uint64_t key_sequence_id{0}; // 该 key 的操作序列号（用于时序保证） -}; -``` - -**JSON 序列化格式**： -```json -{ - "sequence_id": 12345, - "timestamp": 1704110400123, - "op_type": "PUT_END", - "key": "object_key_123", - "payload": "optional_payload", - "checksum": 1234567890, - "prefix_hash": 987654321, - "key_sequence_id": 5 -} -``` - -### 3.5 时序保证机制 - -#### 3.5.1 全局序列号（sequence_id） - -- **作用**：保证所有 OpLog 事件的全局顺序 -- **生成**：Primary 端 `OpLogManager` 全局递增生成 -- **检查**：Standby 端检查 sequence_id 是否连续 - -#### 3.5.2 Key 级别序列号（key_sequence_id） - -- **作用**：保证同一 key 的操作顺序 -- **生成**：Primary 端对每个 key 单独递增 -- **检查**：Standby 端检查 key_sequence_id 是否递增 - -#### 3.5.3 乱序处理 - -当检测到 key_sequence_id 乱序时： -1. **回滚**：从 metadata_store 中删除该 key 的所有状态 -2. **重放**：从该 key 第一次出现的 sequence_id 开始，从 etcd 重新读取所有 OpLog -3. **重写**：按正确顺序重新应用所有 OpLog，重建 metadata - -详细设计请参考：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md` - -### 3.6 快照集成 - -#### 3.6.1 快照时记录 Sequence ID - -- 快照生成时，记录当前的 OpLog sequence_id -- 将快照信息写入 etcd：`mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` - -#### 3.6.2 OpLog 清理 - -- 快照生成后，可以清理快照之前的 OpLog -- 清理策略：查询 etcd 中最小存在的 sequence_id，使用 DeleteRange 删除 - -详细设计请参考：`doc/zh/rfc-oplog-cleanup-start-sequence-id.md` - -### 3.7 Standby 服务集成 - -#### 3.7.1 问题 - -现有代码中，Standby 在等待 leader 选举期间只是阻塞等待，没有运行 Standby 服务来同步 OpLog。 - -#### 3.7.2 解决方案 - -在 `MasterServiceSupervisor::Start()` 中： -1. 检查当前是否有 leader -2. 如果有 leader 且不是自己 → 启动 Standby 服务（watch OpLog 并应用） -3. 选举成功后 → 停止 Standby 服务并提升为 Primary - -详细设计请参考：`doc/zh/rfc-standby-service-integration.md` - -### 3.8 Standby 提升为 Primary 时的 Lease 初始化 - -#### 3.8.1 问题 - -Standby 上的对象 lease 都是 0（因为 OpLog 只包含 PUT_END，不包含续约信息），提升为 Primary 后所有对象会立即过期。 - -#### 3.8.2 解决方案 - -在 `HotStandbyService::Promote()` 时： -1. 停止 Standby 服务 -2. 遍历所有 metadata -3. 对于 lease_timeout = 0 的对象，授予默认租约时间（`default_kv_lease_ttl`） - -详细设计请参考：`doc/zh/rfc-standby-promotion-lease-initialization.md` - -### 3.9 内存优化：key_sequence_map_ 清理 - -#### 3.9.1 问题 - -Standby 端的 `key_sequence_map_` 用于跟踪每个 key 的 `key_sequence_id`。当 metadata 被删除后，这些条目仍然保留，长期运行可能导致内存泄漏。 - -#### 3.9.2 解决方案 - -实现定期清理机制： -- **清理条件**：最后一次操作是 `REMOVE` 且距离当前超过 1 小时 -- **清理频率**：每小时扫描一次 -- **保留策略**：`PUT_END` 和 `PUT_REVOKE` 操作的 key 不清理 - -详细设计请参考：`doc/zh/rfc-oplog-key-sequence-map-cleanup.md` - -## 4. 实施计划 - -详细的实施计划、优先级和时间估算请参考：`doc/zh/rfc-oplog-implementation-plan.md` - -**实施阶段总览**： -- **Phase 1**：基础框架（P0，2-3 周） - - 实现 OpLogManager - - 实现 EtcdOpLogStore - - 实现 OpLogWatcher - - 实现 OpLogApplier - -- **Phase 2**：Standby 服务集成（P0，2-3 周） - - 实现 HotStandbyService - - 集成到 MasterServiceSupervisor - - 实现 Standby 提升为 Primary - -- **Phase 3**：时序保证和容错（P1，2-3 周） - - 实现序列号检查 - - 实现乱序回滚和重放 - - 实现 key_sequence_map_ 清理 - -- **Phase 4**：快照集成和清理（P2，1-2 周） - - 集成快照机制 - - 实现 OpLog 清理 - -- **Phase 5**：优化和完善（P3，1-2 周） - - 批量写入优化 - - 性能调优 - -**总计**：8-13 周（约 2-3 个月） - -## 5. 关键设计要点总结 - -1. **etcd 作为中间存储**：利用 etcd 的强一致性和 Watch 机制 -2. **只记录关键操作**：PUT_END、PUT_REVOKE、REMOVE，不记录 LEASE_RENEW -3. **双重序列号保证**：全局 sequence_id + key 级别 key_sequence_id -4. **快照集成**：与现有快照机制集成，支持 OpLog 清理 -5. **Standby 服务并行运行**：在等待选举期间持续同步数据 -6. **内存优化**：定期清理 key_sequence_map_ 中的过期条目 - -## 6. 相关文档 - -- [OpLog 主备同步完整方案](./rfc-oplog-via-etcd-complete-design.md) -- [Standby 服务集成方案](./rfc-standby-service-integration.md) -- [Standby 提升为 Primary 时的 Lease 初始化](./rfc-standby-promotion-lease-initialization.md) -- [OpLog 序列号乱序时的回滚和重放方案](./rfc-oplog-rollback-replay-on-sequence-violation.md) -- [OpLog 清理策略](./rfc-oplog-cleanup-start-sequence-id.md) -- [key_sequence_map_ 清理策略](./rfc-oplog-key-sequence-map-cleanup.md) -- [实施计划](./rfc-oplog-implementation-plan.md) - diff --git a/doc/zh/rfc-oplog-hot-standby-promotion.md b/doc/zh/rfc-oplog-hot-standby-promotion.md deleted file mode 100644 index 4eb7b6b474..0000000000 --- a/doc/zh/rfc-oplog-hot-standby-promotion.md +++ /dev/null @@ -1,41 +0,0 @@ -# OpLog 主备同步方案 - 宣传文案 - -## 背景动机 - -Mooncake Store 当前高可用模式虽然实现了 Leader 选举，但 Standby Master 在等待期间不执行任何数据同步操作，导致 Primary 故障后 Standby 提升时 metadata 不完整，需要长时间重建，严重影响服务可用性。 - -## 设计亮点 - -**核心创新**：基于 etcd 的 OpLog 主备同步机制 - -1. **可靠的数据同步**：利用 etcd 的强一致性和 Watch 机制，实现 Primary 到 Standby 的实时数据同步，保证 Standby 与 Primary 数据完全一致 - -2. **快速故障恢复**：Standby 持续同步 OpLog，提升为 Primary 时 metadata 完整，无需重建，故障恢复时间从分钟级降低到秒级 - -3. **高效设计**：只记录关键操作（PUT/DELETE），不记录高频的租约续约，OpLog 大小减少 90%+；通过全局和 key 级别双重序列号保证顺序 - -4. **智能容错**：检测到乱序时自动回滚重放，定期清理过期内存，与现有快照机制无缝集成 - -**技术价值**：将高可用模式从不稳定状态提升到生产可用，为 LLM 推理场景提供可靠的高可用保障。 - ---- - -## 群内宣传文案（优化版） - -MoonCake 社区提供了高效的 KV cache 存储方案，极大提高了推理性能，但其高可用性较弱，导致在大规模生产级应用上使用受限。 - -基于此背景，我在社区提出了一种基于热备的高可用架构，已被社区接受。RFC 链接：https://github.com/kvcache-ai/Mooncake/issues/1200 - -**设计亮点**： - -1. **基于 etcd 的 OpLog 机制**：利用强一致性和 Watch 实现实时同步，保证 Standby 与 Primary 数据完全一致 - -2. **秒级故障恢复**：Standby 持续同步，故障恢复从分钟级降至秒级 - -3. **高效设计**：只记录关键操作，OpLog 大小减少 90%+，双重序列号保证顺序 - -4. **智能容错**：乱序自动回滚重放，定期内存清理，与快照机制无缝集成 - -**技术价值**：将高可用模式从基本不可用提升到生产可用，为 LLM 推理提供可靠保障。 - -欢迎大家 review 该 RFC，多提意见哈~ diff --git a/doc/zh/rfc-oplog-implementation-plan.md b/doc/zh/rfc-oplog-implementation-plan.md deleted file mode 100644 index f96d17c754..0000000000 --- a/doc/zh/rfc-oplog-implementation-plan.md +++ /dev/null @@ -1,728 +0,0 @@ -# 基于 etcd 的 OpLog 同步实施计划 - -## 概述 - -本文档基于所有讨论和设计方案，制定了完整的实施计划和优先级。实施计划分为 5 个阶段，从基础框架到优化完善，确保系统逐步稳定地实现 OpLog 同步功能。 - -## 实施阶段总览 - -| 阶段 | 名称 | 优先级 | 预计工作量 | 依赖关系 | -|------|------|--------|-----------|----------| -| **Phase 1** | 基础框架 | **P0（最高）** | 2-3 周 | 无 | -| **Phase 2** | Standby 服务集成 | **P0（最高）** | 2-3 周 | Phase 1 | -| **Phase 3** | 时序保证和容错 | **P1（高）** | 2-3 周 | Phase 1, Phase 2 | -| **Phase 4** | 快照集成和清理 | **P2（中）** | 1-2 周 | Phase 1, Phase 2 | -| **Phase 5** | 优化和完善 | **P3（低）** | 1-2 周 | Phase 1-4 | - -## Phase 1：基础框架（优先级：P0） - -### 目标 -实现 OpLog 写入 etcd 和基础读取功能，为后续功能打下基础。 - -### 任务清单 - -#### 1.1 实现 EtcdOpLogStore（3-4 天） - -**文件**： -- `mooncake-store/include/etcd_oplog_store.h`（已创建） -- `mooncake-store/src/etcd_oplog_store.cpp`（待实现） - -**功能**： -- [ ] `WriteOpLog()`：写入单个 OpLog 到 etcd -- [ ] `ReadOpLog()`：从 etcd 读取单个 OpLog -- [ ] `ReadOpLogSince()`：从指定 sequence_id 开始批量读取 -- [ ] `GetLatestSequenceId()`：获取最新的 sequence_id -- [ ] `RecordSnapshotSequenceId()`：记录快照对应的 sequence_id -- [ ] `GetSnapshotSequenceId()`：获取快照对应的 sequence_id -- [ ] `BuildOpLogKey()`：构建 OpLog key -- [ ] `SerializeOpLogEntry()` / `DeserializeOpLogEntry()`：序列化/反序列化 - -**依赖**： -- `EtcdHelper` 需要支持 `Put`、`Get`、`GetWithPrefix`、`DeleteRange` - -**验收标准**： -- 可以成功写入 OpLog 到 etcd -- 可以成功从 etcd 读取 OpLog -- 支持批量读取（每次最多 1000 条） - -#### 1.2 集成 EtcdOpLogStore 到 OpLogManager（2-3 天） - -**文件**： -- `mooncake-store/src/oplog_manager.cpp`（修改） - -**功能**： -- [ ] 在 `OpLogManager` 中添加 `EtcdOpLogStore` 成员 -- [ ] 在 `Append()` 时调用 `etcd_oplog_store_->WriteOpLog()` -- [ ] 更新 `last_sequence_id_` 到 etcd（可选，用于快速查询） - -**验收标准**： -- Primary 写入 OpLog 时，同时写入 etcd -- 写入失败时有错误处理和日志 - -#### 1.3 在 MasterService 中记录 OpLog（2-3 天） - -**文件**： -- `mooncake-store/src/master_service.cpp`（修改） - -**功能**： -- [ ] `PutEnd()`：记录 `PUT_END` 事件（✅ 已实现） -- [ ] `PutRevoke()`：记录 `PUT_REVOKE` 事件（✅ 已实现） -- [ ] `Remove()`：记录 `REMOVE` 事件（✅ 已实现） -- [ ] `BatchEvict()`：在完全驱逐对象时记录 `REMOVE` 事件（待实现） - -**验收标准**： -- 所有状态变更操作都记录 OpLog -- OpLog 成功写入 etcd - -#### 1.4 实现 etcd Helper 扩展（2-3 天） - -**文件**： -- `mooncake-store/include/etcd_helper.h`（修改） -- `mooncake-store/src/etcd_helper.cpp`（修改） -- `mooncake-store/src/etcd_wrapper.go`（修改） - -**功能**： -- [ ] `GetFirstKeyWithPrefix()`：获取指定前缀的第一个 key（用于 OpLog 清理） -- [ ] `DeleteRange()`：删除指定范围的 key（用于 OpLog 清理） -- [ ] `WatchWithPrefix()`：Watch 指定前缀的 key 变化（用于 OpLog 同步） - -**验收标准**： -- 所有 etcd 操作都有对应的 Helper 方法 -- 错误处理完善 - -### Phase 1 里程碑 - -- ✅ EtcdOpLogStore 可以写入和读取 OpLog -- ✅ Primary 的所有状态变更都写入 etcd -- ✅ etcd Helper 支持所有需要的操作 - -### 测试要求 - -- [ ] 单元测试：EtcdOpLogStore 的读写功能 -- [ ] 集成测试：Primary 写入 OpLog 到 etcd -- [ ] 性能测试：写入性能（目标：> 1000 ops/s） - ---- - -## Phase 2：Standby 服务集成（优先级：P0） - -### 目标 -实现 Standby 服务，使其在等待 leader 选举期间能够 watch etcd OpLog 并实时恢复 metadata。 - -### 任务清单 - -#### 2.1 实现 OpLogWatcher（3-4 天） - -**文件**： -- `mooncake-store/include/oplog_watcher.h`（已创建） -- `mooncake-store/src/oplog_watcher.cpp`（待实现） - -**功能**： -- [ ] `Start()`：启动 Watch 线程 -- [ ] `Stop()`：停止 Watch 线程 -- [ ] `WatchOpLogThreadFunc()`：Watch etcd OpLog 变化 -- [ ] `HandleWatchEvent()`：处理 Watch 事件（PUT/DELETE） -- [ ] `ReadOpLogSince()`：读取历史 OpLog（用于初始同步） - -**依赖**： -- Phase 1.4：`WatchWithPrefix()` 方法 - -**验收标准**： -- 可以成功 Watch etcd OpLog 变化 -- 收到新 OpLog 时调用 `OpLogApplier::ApplyOpLogEntry()` -- 支持断点续传（从上次处理的 sequence_id 继续） - -#### 2.2 实现 OpLogApplier 基础功能（3-4 天） - -**文件**： -- `mooncake-store/include/oplog_applier.h`（已创建） -- `mooncake-store/src/oplog_applier.cpp`（待实现） - -**功能**： -- [ ] `ApplyOpLogEntry()`：应用 OpLog Entry -- [ ] `ApplyPutEnd()`：应用 PUT_END 操作 -- [ ] `ApplyPutRevoke()`：应用 PUT_REVOKE 操作 -- [ ] `ApplyRemove()`：应用 REMOVE 操作 -- [ ] `CheckSequenceOrder()`：检查全局和 key 级别的时序性 -- [ ] `GetLastAppliedSequenceId()`：获取最后应用的 sequence_id - -**依赖**： -- `MetadataStore` 接口（需要定义） - -**验收标准**： -- 可以成功应用 OpLog 到 metadata_store -- 时序检查正确 -- 支持断点续传 - -#### 2.3 修改 HotStandbyService 使用 etcd Watch（2-3 天） - -**文件**： -- `mooncake-store/src/hot_standby_service.cpp`（修改） - -**功能**： -- [ ] 修改 `ReplicationLoop()` 使用 `OpLogWatcher` -- [ ] 先读取历史 OpLog，再启动 Watch -- [ ] 实现 `OpLogApplier` 接口 -- [ ] 处理 Watch 事件并应用 OpLog - -**验收标准**： -- Standby 可以 watch etcd OpLog -- 实时应用 OpLog 到 metadata_store - -#### 2.4 修改 MasterServiceSupervisor 支持 Standby 模式（2-3 天） - -**文件**： -- `mooncake-store/src/ha_helper.cpp`（修改） - -**功能**： -- [ ] 检测到有 leader 时，启动 `HotStandbyService` -- [ ] Standby 服务 watch etcd OpLog 并实时恢复 metadata -- [ ] 选举成功后，停止 Standby 服务并提升为 Primary - -**验收标准**： -- Standby 在等待选举期间持续运行 -- 选举成功后可以正常提升为 Primary - -### Phase 2 里程碑 - -- ✅ Standby 可以 watch etcd OpLog -- ✅ Standby 实时应用 OpLog 到 metadata_store -- ✅ Standby 在等待选举期间持续运行 - -### 测试要求 - -- [ ] 单元测试：OpLogWatcher 和 OpLogApplier -- [ ] 集成测试：Standby watch OpLog 并应用 -- [ ] 端到端测试：Primary 写入，Standby 同步 - ---- - -## Phase 3：时序保证和容错（优先级：P1） - -### 目标 -实现完整的时序保证机制和容错处理，确保数据一致性。 - -### 任务清单 - -#### 3.1 实现序列号不连续处理（2-3 天） - -**文件**： -- `mooncake-store/src/oplog_applier.cpp`（修改） - -**功能**： -- [ ] `ProcessPendingEntries()`：处理待处理的条目 -- [ ] `ScheduleWaitForMissingEntries()`：等待缺失的条目 -- [ ] `RequestMissingOpLog()`：从 etcd 请求缺失的 OpLog -- [ ] 维护 `pending_entries_` 和 `expected_sequence_id_` - -**验收标准**： -- 检测到序列号不连续时，缓存待处理 -- 等待一段时间后，从 etcd 读取缺失的条目 -- 序列号连续后，按顺序应用 - -#### 3.2 实现 key 级别乱序处理（1-2 天） - -**文件**： -- `mooncake-store/src/oplog_applier.cpp`（修改） - -**功能**： -- [x] 检测到 key 级别乱序时，直接删除该 key 的 metadata -- [x] 从 `key_sequence_map_` 中删除该 key -- [x] 删除后继续处理当前 OpLog 条目（如果全局序列号正确） - -**设计说明**： -- 简化方案：不进行回滚和重放，因为前面的数据可能已经丢失 -- 当检测到 `key_sequence_id` 乱序时，直接删除该 key -- 如果后续有 PUT_END 操作，会重新创建该 key -- 这样避免了数据不一致的风险，实现更简单可靠 - -**验收标准**： -- 检测到 key 级别乱序时，正确删除该 key 的 metadata -- 删除后可以继续处理后续的 OpLog 条目 -- 不会导致数据不一致 - -#### 3.3 实现错误处理和恢复（2-3 天） - -**文件**： -- `mooncake-store/src/oplog_applier.cpp`（修改） -- `mooncake-store/src/oplog_watcher.cpp`（修改） -- `mooncake-store/include/oplog_watcher.h`（修改） - -**功能**： -- [x] Watch 断开时自动重连（指数退避策略） -- [x] 重连时同步遗漏的 OpLog 条目（`SyncMissedEntries()`） -- [x] 连续错误计数，超过阈值（10次）时触发重连 -- [x] 重连成功后重置错误计数 -- [x] 完善的日志记录 - -**实现细节**： -- `kMaxConsecutiveErrors = 10`：连续错误超过此阈值触发重连 -- `kReconnectDelayMs = 1000`：初始重连延迟（毫秒） -- `kMaxReconnectDelayMs = 30000`：最大重连延迟（30秒） -- `TryReconnect()`：指数退避重连，重连前同步遗漏条目 -- `SyncMissedEntries()`：从 etcd 读取 `last_processed_sequence_id_` 之后的条目 - -**验收标准**： -- Watch 断开后可以自动重连 -- 重连期间遗漏的 OpLog 可以被正确同步 -- 错误处理完善，不会导致服务崩溃 -- 有完善的日志记录 - -### Phase 3 里程碑 - -- ✅ 序列号不连续时可以正确处理 -- ✅ key 级别乱序时可以回滚和重放 -- ✅ 错误处理和恢复机制完善 - -### 测试要求 - -- [ ] 单元测试：序列号不连续处理 -- [ ] 单元测试：回滚和重放机制 -- [ ] 集成测试：错误恢复场景 -- [ ] 压力测试：大量乱序情况下的性能 - ---- - -## Phase 4：快照集成和清理（优先级：P2）⏸️ 暂缓 - -> **状态**：暂缓，等待与快照团队协调讨论后再实现。 - -### 目标 -集成快照机制，实现 OpLog 清理，减少 etcd 存储压力。 - -### 任务清单 - -#### 4.1 实现快照时记录 sequence_id（2-3 天） - -**文件**： -- `mooncake-store/src/master_service.cpp`（修改） -- 快照相关代码（待确定） - -**功能**： -- [ ] 快照时记录 `last_oplog_sequence_id` -- [ ] 将快照信息写入 etcd（`RecordSnapshotSequenceId()`） -- [ ] Standby 可以从快照点开始同步 - -**验收标准**： -- 快照包含 OpLog 的 sequence_id -- 快照信息可以持久化到 etcd - -#### 4.2 实现 OpLog 清理机制（2-3 天） - -**文件**： -- `mooncake-store/src/etcd_oplog_store.cpp`（修改） - -**功能**： -- [ ] `CleanupOpLogBefore()`：清理指定 sequence_id 之前的 OpLog -- [ ] `GetMinSequenceId()`：从 etcd 查询最小的 sequence_id -- [ ] 使用 `DeleteRange` 批量删除 -- [ ] 定期清理（在快照后或定时任务中） - -**依赖**： -- Phase 1.4：`GetFirstKeyWithPrefix()` 和 `DeleteRange()` - -**验收标准**： -- 可以成功清理旧的 OpLog -- 清理后不影响 Standby 的同步（因为已有快照） - -#### 4.3 实现 Standby 初始同步（2-3 天） - -**文件**： -- `mooncake-store/src/hot_standby_service.cpp`（修改） - -**功能**： -- [ ] 从 Primary 获取快照（或从 etcd 读取最新快照） -- [ ] 应用快照到 metadata_store -- [ ] 从快照的 sequence_id 开始读取增量 OpLog -- [ ] 应用增量 OpLog -- [ ] 启动 Watch 监听新 OpLog - -**验收标准**： -- 新 Standby 可以成功完成初始同步 -- 初始同步后，metadata 与 Primary 一致 - -### Phase 4 里程碑 - -- ✅ 快照时记录 sequence_id -- ✅ 可以清理旧的 OpLog -- ✅ Standby 可以从快照开始同步 - -### 测试要求 - -- [ ] 单元测试：OpLog 清理功能 -- [ ] 集成测试：快照集成 -- [ ] 端到端测试：新 Standby 初始同步 - ---- - -## Phase 5：优化和完善（优先级：P3） - -### 目标 -优化性能，完善功能，提升系统稳定性。 - -### 任务清单 - -#### 5.1 实现 Standby 提升时的 Lease 初始化（2-3 天） - -**文件**： -- `mooncake-store/src/hot_standby_service.cpp`（修改） - -**功能**： -- [ ] `InitializeLeasesForAllObjects()`：给所有 lease 为 0 的对象授予默认租约 -- [ ] `PerformFullEvictionCleanup()`：执行一次完整的驱逐清理 -- [ ] 在 `Promote()` 中调用上述方法 - -**验收标准**： -- Standby 提升为 Primary 时，所有对象都有有效的 lease -- 提升后可以正常执行驱逐 - -#### 5.2 实现批量写入优化（可选，1-2 天） - -**文件**： -- `mooncake-store/src/etcd_oplog_store.cpp`（修改） - -**功能**： -- [ ] `WriteOpLogBatch()`：批量写入 OpLog -- [ ] 使用事务保证原子性 -- [ ] 减少 etcd 写入次数 - -**验收标准**： -- 批量写入性能提升 -- 不影响数据一致性 - -#### 5.3 实现 OpLog 压缩（可选，1-2 天） - -**文件**： -- `mooncake-store/src/etcd_oplog_store.cpp`（修改） - -**功能**： -- [ ] OpLog Entry 压缩（如使用 gzip） -- [ ] 减少 etcd 存储大小 - -**验收标准**： -- 压缩后存储大小减少 -- 不影响读取性能 - -#### 5.4 完善监控和告警（1-2 天） - -**功能**： -- [ ] OpLog 写入速率监控 -- [ ] Standby 同步延迟监控 -- [ ] 乱序频率监控 -- [ ] 回滚次数和耗时监控 -- [ ] 告警机制（超过阈值时告警） - -**验收标准**： -- 所有关键指标都有监控 -- 有完善的告警机制 - -### Phase 5 里程碑 - -- ✅ Standby 提升时 lease 初始化完成 -- ✅ 性能优化完成 -- ✅ 监控和告警完善 - -### 测试要求 - -- [ ] 单元测试：Lease 初始化 -- [ ] 性能测试：批量写入和压缩效果 -- [ ] 监控测试：监控指标正确 - ---- - -## 依赖关系图 - -``` -Phase 1: 基础框架 - ├─ 1.1 EtcdOpLogStore - ├─ 1.2 集成到 OpLogManager - ├─ 1.3 MasterService 记录 OpLog - └─ 1.4 etcd Helper 扩展 - │ - ▼ -Phase 2: Standby 服务集成 - ├─ 2.1 OpLogWatcher ──────┐ - ├─ 2.2 OpLogApplier ──────┤ - ├─ 2.3 HotStandbyService ─┤ - └─ 2.4 MasterServiceSupervisor ─┐ - │ │ - ▼ │ -Phase 3: 时序保证和容错 │ - ├─ 3.1 序列号不连续处理 │ - ├─ 3.2 回滚和重放机制 │ - └─ 3.3 错误处理和恢复 │ - │ │ - ▼ │ -Phase 4: 快照集成和清理 │ - ├─ 4.1 快照记录 sequence_id │ - ├─ 4.2 OpLog 清理 ───────────────┘ - └─ 4.3 Standby 初始同步 - │ - ▼ -Phase 5: 优化和完善 - ├─ 5.1 Lease 初始化 - ├─ 5.2 批量写入优化（可选） - ├─ 5.3 OpLog 压缩（可选） - └─ 5.4 监控和告警 -``` - -## 关键里程碑 - -| 里程碑 | 阶段 | 验收标准 | -|--------|------|----------| -| **M1** | Phase 1 完成 | Primary 可以写入 OpLog 到 etcd | -| **M2** | Phase 2 完成 | Standby 可以 watch OpLog 并实时同步 | -| **M3** | Phase 3 完成 | 时序保证和容错机制完善 | -| **M4** | Phase 4 完成 | 快照集成和 OpLog 清理完成 | -| **M5** | Phase 5 完成 | 所有优化和完善完成 | - -## 风险评估 - -### 高风险项 - -1. **etcd 性能瓶颈** - - **风险**：大量 OpLog 写入可能导致 etcd 性能下降 - - **缓解**：批量写入、压缩、定期清理 - - **监控**：etcd 写入速率、延迟、存储大小 - -2. **Watch 断开和重连** - - **风险**：Watch 断开可能导致数据丢失 - - **缓解**：自动重连、断点续传、从 etcd 重新读取 - - **监控**：Watch 断开次数、重连时间 - -3. **序列号乱序** - - **风险**：乱序可能导致数据不一致 - - **缓解**：回滚和重放机制、监控告警 - - **监控**：乱序频率、回滚次数 - -### 中风险项 - -1. **Standby 提升时的数据迁移** - - **风险**：metadata 迁移可能失败 - - **缓解**：完善的错误处理、回滚机制 - - **监控**：提升成功率、迁移耗时 - -2. **快照和 OpLog 的一致性** - - **风险**：快照和 OpLog 可能不一致 - - **缓解**：快照时记录 sequence_id、验证机制 - - **监控**：快照和 OpLog 的一致性检查 - -## 测试策略 - -### 单元测试 - -- [ ] EtcdOpLogStore 的所有方法 -- [ ] OpLogWatcher 的 Watch 功能 -- [ ] OpLogApplier 的应用逻辑 -- [ ] 时序检查逻辑 -- [ ] 回滚和重放逻辑 - -### 集成测试 - -- [ ] Primary 写入 → etcd → Standby 同步 -- [ ] Standby 初始同步（快照 + OpLog） -- [ ] Standby 提升为 Primary -- [ ] OpLog 清理机制 -- [ ] 错误恢复场景 - -### 端到端测试 - -- [ ] 完整的主备切换流程 -- [ ] 长时间运行稳定性测试 -- [ ] 高负载下的性能测试 -- [ ] 故障注入测试 - -### 性能测试 - -- [ ] OpLog 写入性能（目标：> 1000 ops/s） -- [ ] Standby 同步延迟（目标：< 100ms） -- [ ] etcd 存储大小（目标：10 分钟内 < 1GB） -- [ ] 回滚和重放性能 - -## 文档要求 - -### 必须完成的文档 - -- [x] 主设计文档：`doc/zh/rfc-oplog-via-etcd-complete-design.md` -- [x] Standby 服务集成：`doc/zh/rfc-standby-service-integration.md` -- [x] Lease 初始化：`doc/zh/rfc-standby-promotion-lease-initialization.md` -- [x] OpLog 清理：`doc/zh/rfc-oplog-cleanup-start-sequence-id.md` -- [x] 回滚和重放：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md` -- [x] 实施计划：`doc/zh/rfc-oplog-implementation-plan.md`（本文档） - -### 可选文档 - -- [ ] API 文档：各个类的接口说明 -- [ ] 运维文档：部署和运维指南 -- [ ] 故障排查文档：常见问题和解决方案 - -## 时间估算 - -### 总体时间 - -- **Phase 1**：2-3 周（P0） -- **Phase 2**：2-3 周（P0） -- **Phase 3**：2-3 周（P1） -- **Phase 4**：1-2 周（P2） -- **Phase 5**：1-2 周（P3） - -**总计**：8-13 周（约 2-3 个月） - -### 关键路径 - -``` -Phase 1 → Phase 2 → Phase 3 → Phase 4 → Phase 5 -``` - -**最短时间**：8 周（如果所有阶段都按最短时间完成） - -### 并行开发可能性 - -- **Phase 1 和 Phase 2**：可以部分并行（Phase 2 的 OpLogApplier 可以在 Phase 1 完成后开始） -- **Phase 3 和 Phase 4**：可以部分并行（快照集成和时序保证相对独立） -- **Phase 5**：可以在 Phase 1-4 完成后开始 - -## 优先级说明 - -### P0（最高优先级） - -- **Phase 1**：基础框架，所有后续功能都依赖于此 -- **Phase 2**：Standby 服务集成，核心功能 - -**必须完成**：这两个阶段是核心功能，必须优先完成。 - -### P1（高优先级） - -- **Phase 3**：时序保证和容错，确保数据一致性 - -**重要**：这个阶段确保数据一致性，应该在 Phase 1-2 完成后尽快完成。 - -### P2（中优先级） - -- **Phase 4**：快照集成和清理，减少存储压力 - -**可选但推荐**：这个阶段可以减少 etcd 存储压力，建议完成。 - -### P3（低优先级） - -- **Phase 5**：优化和完善，提升系统稳定性 - -**可选**：这个阶段是优化，可以在系统稳定运行后再完成。 - -## 实施建议 - -### 第一步：完成 Phase 1 - -1. **先实现 `EtcdOpLogStore` 的基础功能**（写入和读取） - - 确保可以成功写入和读取 OpLog - - 完成单元测试 - -2. **集成到 `OpLogManager`** - - 确保 Primary 可以写入 OpLog - - 完成集成测试 - -3. **扩展 `EtcdHelper`** - - 支持所有需要的操作 - - 完成单元测试 - -4. **完成测试** - - 单元测试、集成测试、性能测试 - -### 第二步：完成 Phase 2 - -1. **实现 `OpLogWatcher`** - - 支持 Watch etcd - - 完成单元测试 - -2. **实现 `OpLogApplier` 基础功能** - - 可以应用 OpLog - - 完成单元测试 - -3. **修改 `HotStandbyService`** - - 使用 etcd Watch - - 完成集成测试 - -4. **修改 `MasterServiceSupervisor`** - - 支持 Standby 模式 - - 完成端到端测试 - -### 第三步：完成 Phase 3 - -1. **实现序列号不连续处理** - - 缓存待处理条目 - - 从 etcd 读取缺失条目 - -2. **实现回滚和重放机制** - - 检测乱序 - - 回滚和重放 - -3. **完善错误处理和恢复** - - Watch 重连 - - 错误重试 - -4. **完成压力测试** - -### 第四步：完成 Phase 4 和 Phase 5 - -1. **实现快照集成和 OpLog 清理** - - 快照时记录 sequence_id - - 清理旧的 OpLog - -2. **实现 Standby 提升时的 Lease 初始化** - - 初始化所有对象的 lease - - 执行驱逐清理 - -3. **优化性能和完善监控** - - 批量写入（可选） - - OpLog 压缩（可选） - - 监控和告警 - -4. **完成所有测试** - -## 关键成功因素 - -### 1. 代码质量 - -- **代码审查**：每个阶段完成后进行代码审查 -- **单元测试覆盖率**：目标 > 80% -- **集成测试**：确保各组件正确集成 - -### 2. 性能要求 - -- **OpLog 写入性能**：> 1000 ops/s -- **Standby 同步延迟**：< 100ms -- **etcd 存储大小**：10 分钟内 < 1GB - -### 3. 稳定性要求 - -- **错误处理**：所有错误都有完善的处理 -- **自动恢复**：Watch 断开、读取失败等可以自动恢复 -- **监控告警**：关键指标都有监控和告警 - -### 4. 文档要求 - -- **设计文档**：所有设计都有详细文档 -- **API 文档**：所有接口都有文档 -- **运维文档**：部署和运维指南 - -## 总结 - -本实施计划按照依赖关系和重要性，将整个项目分为 5 个阶段。**Phase 1 和 Phase 2 是核心功能，必须优先完成**。Phase 3 确保数据一致性，Phase 4 和 Phase 5 是优化和完善。 - -**建议按照阶段顺序实施，每个阶段完成后进行充分测试，确保稳定性后再进入下一阶段。** - -### 关键要点 - -1. **优先级明确**：P0 > P1 > P2 > P3 -2. **依赖关系清晰**：Phase 1 → Phase 2 → Phase 3 → Phase 4 → Phase 5 -3. **测试充分**：每个阶段都有对应的测试要求 -4. **风险可控**：识别了高风险项并提供了缓解措施 -5. **时间合理**：总计 8-13 周，符合项目时间要求 - -### 下一步行动 - -1. **评审本计划**：与团队评审实施计划 -2. **分配任务**：根据计划分配开发任务 -3. **开始 Phase 1**：从基础框架开始实施 -4. **定期检查**：每周检查进度，确保按计划进行 - diff --git a/doc/zh/rfc-oplog-key-sequence-map-cleanup.md b/doc/zh/rfc-oplog-key-sequence-map-cleanup.md deleted file mode 100644 index 6d954d46dc..0000000000 --- a/doc/zh/rfc-oplog-key-sequence-map-cleanup.md +++ /dev/null @@ -1,253 +0,0 @@ -# OpLogApplier key_sequence_map_ 清理策略 - -## 问题描述 - -在 Standby 端的 `OpLogApplier` 中，`key_sequence_map_` 用于跟踪每个 key 的 `key_sequence_id`，以确保 OpLog 的顺序正确性。当 metadata 被删除（REMOVE 操作）后，`key_sequence_map_` 中的条目仍然保留，用于检测可能的乱序操作。 - -### 内存泄漏风险 - -如果 `key_sequence_map_` 中的条目一直不删除，长期运行可能导致内存泄漏： - -- **内存占用**：每个条目约 90 字节（string key + uint64_t value + hash map 开销） -- **累积效应**：系统长期运行，可能有数百万个不同的 key 曾经存在过 -- **极端场景**：如果每天创建 10 万个新 key，运行 100 天，累计 1000 万个不同的 key，内存占用可达 900MB - -### 清理需求 - -需要在保证功能正确性的前提下，实现内存清理机制。 - -## 解决方案 - -### 核心策略 - -**清理条件**： -1. 最后一次操作是 `REMOVE`（DELETE） -2. 距离当前时间超过 1 小时 - -**清理频率**：每小时扫描一次 - -**保留策略**： -- `PUT_END` 和 `PUT_REVOKE` 操作的 key 不清理（metadata 可能仍存在） -- 即使超过 1 小时，只要最后操作不是 `REMOVE`，也保留 - -### 设计原理 - -1. **乱序检测时间窗口**：乱序检测一般只需要秒级的时间窗口，1 小时的保留时间足够处理网络延迟、重传等情况 -2. **只清理 DELETE 操作**：因为 DELETE 操作的 metadata 已经不存在，且超过 1 小时后不太可能再出现乱序 -3. **保留 PUT 操作**：PUT 操作的 metadata 可能仍存在，需要保留用于顺序检查 - -## 实现设计 - -### 数据结构 - -```cpp -class OpLogApplier { -private: - struct KeySequenceInfo { - uint64_t sequence_id{0}; - OpType last_op_type{OpType::PUT_END}; - std::chrono::steady_clock::time_point last_op_time; - - KeySequenceInfo() - : last_op_time(std::chrono::steady_clock::now()) {} - }; - - std::unordered_map key_sequence_map_; - mutable std::mutex key_sequence_mutex_; - - // 清理配置 - static constexpr std::chrono::hours kCleanupInterval{1}; // 每小时清理一次 - static constexpr std::chrono::hours kStaleThreshold{1}; // 1小时未访问则清理（仅限DELETE） - - std::chrono::steady_clock::time_point last_cleanup_time_; -}; -``` - -### 核心方法 - -#### 1. 定期清理检查 - -```cpp -void OpLogApplier::PeriodicCleanup() { - auto now = std::chrono::steady_clock::now(); - if (now - last_cleanup_time_ < kCleanupInterval) { - return; // 还没到清理时间 - } - - CleanupStaleKeySequences(); - last_cleanup_time_ = now; -} -``` - -#### 2. 清理过期条目 - -```cpp -void OpLogApplier::CleanupStaleKeySequences() { - std::lock_guard lock(key_sequence_mutex_); - auto now = std::chrono::steady_clock::now(); - auto threshold = now - kStaleThreshold; - - size_t cleaned = 0; - for (auto it = key_sequence_map_.begin(); - it != key_sequence_map_.end();) { - const auto& info = it->second; - - // 清理条件： - // 1. 最后一次操作是 REMOVE（DELETE） - // 2. 且距离当前超过1小时 - if (info.last_op_type == OpType::REMOVE && - info.last_op_time < threshold) { - it = key_sequence_map_.erase(it); - cleaned++; - } else { - ++it; - } - } - - if (cleaned > 0) { - LOG(INFO) << "Cleaned up " << cleaned - << " stale key_sequence_map entries " - << "(REMOVE operations older than 1 hour)"; - } -} -``` - -#### 3. 应用 OpLog 时更新 - -```cpp -bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { - // 1. 检查顺序 - if (!CheckSequenceOrder(entry)) { - // 处理乱序... - return false; - } - - // 2. 应用操作 - switch (entry.op_type) { - case OpType::PUT_END: - ApplyPutEnd(entry); - break; - case OpType::PUT_REVOKE: - ApplyPutRevoke(entry); - break; - case OpType::REMOVE: - ApplyRemove(entry); - break; - } - - // 3. 更新 key_sequence_map_ - { - std::lock_guard lock(key_sequence_mutex_); - auto& info = key_sequence_map_[entry.object_key]; - info.sequence_id = entry.key_sequence_id; - info.last_op_type = entry.op_type; - info.last_op_time = std::chrono::steady_clock::now(); - } - - // 4. 定期清理（每次应用时检查，避免额外线程） - PeriodicCleanup(); - - return true; -} -``` - -## 关键设计要点 - -### 1. 清理时机 - -- **触发方式**：在 `ApplyOpLogEntry` 中检查，无需额外线程 -- **清理频率**：每小时执行一次 -- **清理条件**：只清理 `REMOVE` 操作且超过 1 小时的条目 - -### 2. 安全性保证 - -- **保留 PUT 操作**：`PUT_END` 和 `PUT_REVOKE` 的 key 不清理，因为 metadata 可能仍存在 -- **1 小时窗口**：足够处理网络延迟、重传等异常情况 -- **线程安全**：使用 mutex 保护 `key_sequence_map_` 的访问 - -### 3. 内存占用控制 - -**清理前**： -- 假设系统长期运行，有 100 万个不同的 key 曾经存在过 -- 内存占用：100万 × 90字节 ≈ 90MB - -**清理后**： -- 假设系统每小时处理 10 万个 OpLog，其中 10% 是 REMOVE 操作 -- `key_sequence_map_` 中最多保留： - - 最近 1 小时的 REMOVE key：约 1 万个 - - 所有 PUT_END/PUT_REVOKE 的 key：取决于实际 metadata 数量 -- 内存占用：约 `(活跃key数量 + 1万) × 90字节` - -**内存节省**：从 90MB 降低到约 `(活跃key数量 + 1万) × 90字节`，通常远小于不清理的情况。 - -## 使用场景示例 - -### 场景 1：正常 REMOVE 操作 - -``` -时间线： -1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5, op_type=PUT_END - → 应用成功，key_sequence_map_["obj1"] = {seq:5, op:PUT_END, time:10:00} - -2. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=6, op_type=REMOVE - → 应用成功，key_sequence_map_["obj1"] = {seq:6, op:REMOVE, time:10:05} - → metadata 被删除 - -3. 1小时后（11:05），清理扫描 - → 检测到 "obj1" 的 last_op_type=REMOVE 且超过1小时 - → 清理 key_sequence_map_["obj1"] -``` - -### 场景 2：乱序 REMOVE 操作 - -``` -时间线： -1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5, op_type=PUT_END - → 应用成功，key_sequence_map_["obj1"] = {seq:5, op:PUT_END, time:10:00} - -2. Standby 收到 OpLog: sequence_id=102, key="obj1", key_sequence_id=6, op_type=PUT_END - → 应用成功，key_sequence_map_["obj1"] = {seq:6, op:PUT_END, time:10:02} - -3. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=5, op_type=REMOVE - → 检测到乱序：entry.key_sequence_id(5) <= current(6) - → 触发回滚和重放 - → key_sequence_map_["obj1"] 保留用于重放 -``` - -### 场景 3：删除后重新创建 - -``` -时间线： -1. key="obj1" 被 REMOVE，key_sequence_map_["obj1"] = {seq:6, op:REMOVE, time:10:00} - -2. 30分钟后（10:30），Standby 收到 OpLog: sequence_id=200, key="obj1", key_sequence_id=7, op_type=PUT_END - → 检查：key_sequence_map_["obj1"] 存在，seq=6（期望） - → 应用成功，key_sequence_map_["obj1"] = {seq:7, op:PUT_END, time:10:30} - → 不会被清理（因为 last_op_type=PUT_END） -``` - -## 配置参数 - -| 参数 | 默认值 | 说明 | -|------|--------|------| -| `kCleanupInterval` | 1 小时 | 清理检查的间隔时间 | -| `kStaleThreshold` | 1 小时 | REMOVE 操作超过此时间后可以清理 | - -## 优势 - -1. **内存控制**：及时清理已删除且超过 1 小时的 key，有效控制内存占用 -2. **安全性**：保留最近删除的 key，确保乱序检测的正确性 -3. **简单高效**：无需额外线程，在应用 OpLog 时检查，实现简单 -4. **精确清理**：只清理符合条件的条目，不影响活跃 key - -## 注意事项 - -1. **清理时机**：清理在 `ApplyOpLogEntry` 中触发，如果长时间没有 OpLog，可能不会及时清理 -2. **时间精度**：使用 `std::chrono::steady_clock`，不受系统时间调整影响 -3. **线程安全**：所有对 `key_sequence_map_` 的访问都需要加锁保护 - -## 相关文档 - -- [OpLog 主备同步完整方案](./rfc-oplog-via-etcd-complete-design.md) -- [OpLog 序列号乱序时的回滚和重放方案](./rfc-oplog-rollback-replay-on-sequence-violation.md) - diff --git a/doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md b/doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md deleted file mode 100644 index 5b3338521f..0000000000 --- a/doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md +++ /dev/null @@ -1,653 +0,0 @@ -# OpLog 序列号乱序时的回滚和重放方案 - -## 问题描述 - -当 Standby 检测到某个 key 的 `key_sequence_id` 乱序时（例如：收到了 `key_sequence_id=5`，但之前已经处理了 `key_sequence_id=6`），说明该 key 的 metadata 可能已经不一致。 - -### 问题场景 - -``` -时间线： -1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5, op_type=PUT_END -2. Standby 应用成功，metadata 中 obj1 的 key_sequence_id = 5 -3. Standby 收到 OpLog: sequence_id=102, key="obj1", key_sequence_id=6, op_type=PUT_END -4. Standby 应用成功，metadata 中 obj1 的 key_sequence_id = 6 -5. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=5, op_type=REMOVE - ❌ 乱序！key_sequence_id=5 < 当前值 6 -``` - -**问题**： -- 该 key 的 metadata 可能已经不一致 -- 需要修复该 key 的数据状态 - -## 解决方案 - -### 核心思路 - -**对于乱序的 key，执行回滚和重放**： -1. **回滚**：从 metadata_store 中删除该 key 的所有状态 -2. **重放**：从该 key 第一次出现的 sequence_id 开始，从 etcd 重新读取所有 OpLog -3. **重写**：按正确顺序重新应用所有 OpLog，重建 metadata - -### 架构设计 - -``` -┌─────────────────────────────────────────────────────────┐ -│ OpLogApplier::ApplyOpLogEntry() │ -└─────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 检查 key_sequence_id 是否递增 │ -│ - 如果乱序 → 触发回滚和重放 │ -└─────────────────────────────────────────────────────────┘ - │ - ├─ 正常顺序 - │ │ - │ ▼ - │ ┌─────────────────────────────────────────┐ - │ │ 正常应用 OpLog │ - │ └─────────────────────────────────────────┘ - │ - └─ 乱序 - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 1. 回滚：删除该 key 的 metadata │ -│ - metadata_store_->RemoveKey(key) │ -└─────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 2. 从 etcd 重新读取该 key 的所有 OpLog │ -│ - ReadOpLogForKey(key, first_seq_id) │ -│ - 过滤出该 key 的条目 │ -│ - 按 sequence_id 排序 │ -└─────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 3. 按顺序重新应用所有 OpLog │ -│ - 跳过时序检查（因为已经排序） │ -│ - 重新构建 metadata │ -└─────────────────────────────────────────────────────────┘ -``` - -## 实现设计 - -### 1. 方案 A：基于 etcd 的完整重放（推荐） - -**优点**： -- 数据准确：从 etcd 读取保证数据正确 -- 实现简单：不需要维护操作历史 -- 内存友好：不需要额外存储 -- 容错性好：即使本地状态丢失也能恢复 - -**缺点**： -- 需要从 etcd 读取：可能有网络 I/O 开销 -- 可能较慢：如果该 key 的操作很多 - -#### 实现代码 - -```cpp -class OpLogApplier { -private: - // 记录每个 key 的首次 sequence_id（用于回滚） - std::unordered_map key_first_sequence_id_; - std::mutex key_first_sequence_mutex_; - - // 记录正在回滚的 key（防止并发回滚） - std::set keys_under_rollback_; - std::mutex rollback_mutex_; - - EtcdOpLogStore* etcd_oplog_store_; - -public: - bool ApplyOpLogEntry(const OpLogEntry& entry) { - // 1. 记录首次 sequence_id - { - std::lock_guard lock(key_first_sequence_mutex_); - if (key_first_sequence_id_.count(entry.object_key) == 0) { - key_first_sequence_id_[entry.object_key] = entry.sequence_id; - } - } - - // 2. 检查 key 级别的时序性 - if (!CheckSequenceOrder(entry)) { - LOG(ERROR) << "Key-level sequence order violation for key: " - << entry.object_key - << ", entry_seq=" << entry.key_sequence_id - << ", current_seq=" << GetKeySequenceId(entry.object_key); - - // 3. 触发回滚和重放（异步执行，不阻塞） - std::thread([this, key = entry.object_key]() { - RollbackAndReplayKey(key); - }).detach(); - - // 暂时跳过这个条目，等待回滚完成 - return false; - } - - // 4. 正常应用 - switch (entry.op_type) { - case OpType::PUT_END: - ApplyPutEnd(entry); - break; - case OpType::PUT_REVOKE: - ApplyPutRevoke(entry); - break; - case OpType::REMOVE: - ApplyRemove(entry); - break; - } - - // 5. 更新 key_sequence_map_ - { - std::lock_guard lock(key_sequence_mutex_); - key_sequence_map_[entry.object_key] = entry.key_sequence_id; - } - - return true; - } - -private: - bool RollbackAndReplayKey(const std::string& key) { - // 1. 检查是否正在回滚（防止并发回滚） - { - std::lock_guard lock(rollback_mutex_); - if (keys_under_rollback_.count(key) > 0) { - LOG(WARNING) << "Key is already under rollback: " << key; - return false; - } - keys_under_rollback_.insert(key); - } - - // 2. 获取该 key 的首次 sequence_id - uint64_t first_seq_id; - { - std::lock_guard lock(key_first_sequence_mutex_); - auto it = key_first_sequence_id_.find(key); - if (it == key_first_sequence_id_.end()) { - LOG(ERROR) << "Cannot find first sequence_id for key: " << key; - std::lock_guard lock2(rollback_mutex_); - keys_under_rollback_.erase(key); - return false; - } - first_seq_id = it->second; - } - - // 3. 回滚：从 metadata_store_ 中删除该 key - LOG(INFO) << "Rolling back key: " << key - << ", removing from metadata_store"; - metadata_store_->RemoveKey(key); - - // 4. 从 etcd 重新读取该 key 的所有 OpLog - LOG(INFO) << "Re-reading OpLog for key: " << key - << " from sequence_id: " << first_seq_id; - - std::vector key_entries; - if (!ReadOpLogForKey(key, first_seq_id, key_entries)) { - LOG(ERROR) << "Failed to read OpLog for key: " << key; - std::lock_guard lock(rollback_mutex_); - keys_under_rollback_.erase(key); - return false; - } - - // 5. 按顺序重新应用所有 OpLog - LOG(INFO) << "Replaying " << key_entries.size() - << " OpLog entries for key: " << key; - - for (const auto& entry : key_entries) { - // 重新应用（跳过时序检查，因为我们已经从 etcd 读取了正确的顺序） - switch (entry.op_type) { - case OpType::PUT_END: - ApplyPutEnd(entry); - break; - case OpType::PUT_REVOKE: - ApplyPutRevoke(entry); - break; - case OpType::REMOVE: - ApplyRemove(entry); - break; - } - - // 更新 key_sequence_map_ - { - std::lock_guard lock(key_sequence_mutex_); - key_sequence_map_[key] = entry.key_sequence_id; - } - } - - // 6. 清除回滚标记 - { - std::lock_guard lock(rollback_mutex_); - keys_under_rollback_.erase(key); - } - - LOG(INFO) << "Successfully replayed OpLog for key: " << key; - return true; - } - - bool ReadOpLogForKey(const std::string& key, - uint64_t start_seq_id, - std::vector& entries) { - // 从 etcd 读取从 start_seq_id 开始的所有 OpLog - std::vector all_entries; - const uint32_t batch_size = 10000; // 批量读取 - - uint64_t current_seq_id = start_seq_id; - while (true) { - std::vector batch; - if (!etcd_oplog_store_->ReadOpLogSince(current_seq_id, batch_size, batch)) { - LOG(ERROR) << "Failed to read OpLog from etcd"; - return false; - } - - if (batch.empty()) { - break; // 没有更多条目 - } - - // 过滤出该 key 的条目 - for (const auto& entry : batch) { - if (entry.object_key == key) { - entries.push_back(entry); - } - } - - // 更新 current_seq_id - if (batch.size() < batch_size) { - break; // 已读取完所有条目 - } - current_seq_id = batch.back().sequence_id + 1; - } - - // 按 sequence_id 排序（确保顺序正确） - std::sort(entries.begin(), entries.end(), - [](const OpLogEntry& a, const OpLogEntry& b) { - return a.sequence_id < b.sequence_id; - }); - - return true; - } -}; -``` - -### 2. 方案 B：基于操作历史的回滚（可选） - -**优点**： -- 快速：不需要网络 I/O -- 高效：直接从内存读取 - -**缺点**： -- 需要额外内存：存储操作历史 -- 实现复杂：需要维护历史记录 -- 容错性差：如果历史丢失，无法恢复 - -#### 实现代码 - -```cpp -class OpLogApplier { -private: - // 记录每个 key 的操作历史（用于回滚） - struct KeyOperationHistory { - std::vector operations; // 按顺序记录的操作 - uint64_t first_sequence_id{0}; - }; - std::unordered_map key_history_; - std::mutex key_history_mutex_; - - // 限制历史记录的大小（避免内存无限增长） - static constexpr size_t kMaxHistorySize = 1000; - -public: - bool ApplyOpLogEntry(const OpLogEntry& entry) { - // 1. 记录操作历史 - { - std::lock_guard lock(key_history_mutex_); - auto& history = key_history_[entry.object_key]; - if (history.operations.empty()) { - history.first_sequence_id = entry.sequence_id; - } - - // 限制历史记录大小 - if (history.operations.size() < kMaxHistorySize) { - history.operations.push_back(entry); - } else { - // 如果超过限制，只保留最近的操作 - history.operations.erase(history.operations.begin()); - history.operations.push_back(entry); - } - } - - // 2. 检查时序性 - if (!CheckSequenceOrder(entry)) { - return RollbackAndReplayKey(entry.object_key); - } - - // 3. 正常应用 - // ... - } - -private: - bool RollbackAndReplayKey(const std::string& key) { - std::lock_guard lock(key_history_mutex_); - - auto it = key_history_.find(key); - if (it == key_history_.end()) { - LOG(ERROR) << "Cannot find history for key: " << key; - return false; - } - - // 1. 回滚：删除该 key 的 metadata - metadata_store_->RemoveKey(key); - - // 2. 重新应用所有操作（从历史记录中） - for (const auto& entry : it->second.operations) { - // 重新应用 - switch (entry.op_type) { - case OpType::PUT_END: - ApplyPutEnd(entry); - break; - case OpType::PUT_REVOKE: - ApplyPutRevoke(entry); - break; - case OpType::REMOVE: - ApplyRemove(entry); - break; - } - - // 更新 key_sequence_map_ - { - std::lock_guard lock2(key_sequence_mutex_); - key_sequence_map_[key] = entry.key_sequence_id; - } - } - - return true; - } -}; -``` - -## 关键设计点 - -### 1. 回滚起点的确定 - -**方案 A（推荐）**： -- 维护 `key_first_sequence_id_` 记录每个 key 第一次出现的 sequence_id -- 从该 sequence_id 开始重新读取所有 OpLog - -**方案 B**： -- 维护操作历史，从历史记录中获取所有操作 - -### 2. 并发处理 - -**问题**：回滚期间，如果收到新的 OpLog 怎么办？ - -**解决方案**： -- 使用 `keys_under_rollback_` 标记正在回滚的 key -- 回滚期间，新的 OpLog 暂时跳过（返回 false） -- 回滚完成后，新的 OpLog 可以正常处理 - -```cpp -bool ApplyOpLogEntry(const OpLogEntry& entry) { - // 检查是否正在回滚 - { - std::lock_guard lock(rollback_mutex_); - if (keys_under_rollback_.count(entry.object_key) > 0) { - LOG(WARNING) << "Key is under rollback, skipping entry: " - << entry.sequence_id; - return false; // 暂时跳过，等待回滚完成 - } - } - - // 正常处理 - // ... -} -``` - -### 3. 性能优化 - -#### 3.1 异步回滚 - -**问题**：回滚和重放可能耗时，会阻塞新 OpLog 的处理 - -**解决方案**：异步执行回滚，不阻塞正常处理 - -```cpp -if (!CheckSequenceOrder(entry)) { - // 异步回滚（不阻塞） - std::thread([this, key = entry.object_key]() { - RollbackAndReplayKey(key); - }).detach(); - - return false; // 暂时跳过 -} -``` - -#### 3.2 批量读取 - -**问题**：从 etcd 读取大量 OpLog 可能较慢 - -**解决方案**：批量读取，减少网络往返 - -```cpp -bool ReadOpLogForKey(const std::string& key, - uint64_t start_seq_id, - std::vector& entries) { - const uint32_t batch_size = 10000; // 批量读取 - uint64_t current_seq_id = start_seq_id; - - while (true) { - std::vector batch; - etcd_oplog_store_->ReadOpLogSince(current_seq_id, batch_size, batch); - // ... - } -} -``` - -#### 3.3 限制回滚范围 - -**问题**：如果该 key 的操作非常多，回滚可能很耗时 - -**解决方案**：限制回滚范围，只回滚最近的操作 - -```cpp -bool RollbackAndReplayKey(const std::string& key) { - // 只回滚最近 N 个操作 - const uint64_t max_rollback_ops = 1000; - - // 从 etcd 读取时，限制范围 - uint64_t start_seq_id = std::max( - first_seq_id, - GetLatestSequenceId() - max_rollback_ops - ); - - // ... -} -``` - -### 4. 错误处理 - -#### 4.1 回滚失败 - -**场景**：从 etcd 读取 OpLog 失败 - -**处理**： -- 记录错误日志 -- 清除回滚标记 -- 可以考虑触发全量同步 - -```cpp -if (!ReadOpLogForKey(key, first_seq_id, key_entries)) { - LOG(ERROR) << "Failed to read OpLog for key: " << key; - - // 清除回滚标记 - { - std::lock_guard lock(rollback_mutex_); - keys_under_rollback_.erase(key); - } - - // 可选：触发全量同步 - // TriggerFullSync(); - - return false; -} -``` - -#### 4.2 重复回滚 - -**场景**：同一个 key 多次触发回滚 - -**处理**： -- 使用 `keys_under_rollback_` 防止并发回滚 -- 如果正在回滚，跳过新的回滚请求 - -### 5. 监控和告警 - -#### 5.1 记录乱序频率 - -```cpp -class OpLogApplier { -private: - // 记录每个 key 的乱序次数 - std::unordered_map key_violation_count_; - std::mutex violation_count_mutex_; - - // 乱序阈值 - static constexpr uint64_t kMaxViolationsPerKey = 10; - -public: - bool ApplyOpLogEntry(const OpLogEntry& entry) { - if (!CheckSequenceOrder(entry)) { - // 记录乱序次数 - { - std::lock_guard lock(violation_count_mutex_); - key_violation_count_[entry.object_key]++; - - if (key_violation_count_[entry.object_key] > kMaxViolationsPerKey) { - LOG(ERROR) << "Too many violations for key: " - << entry.object_key - << ", count: " - << key_violation_count_[entry.object_key]; - - // 触发全量同步 - TriggerFullSync(); - return false; - } - } - - // 触发回滚 - // ... - } - } -}; -``` - -#### 5.2 性能指标 - -- 回滚次数 -- 回滚耗时 -- 回滚成功率 -- 乱序频率 - -## 方案对比 - -| 特性 | 方案 A（基于 etcd） | 方案 B（基于历史） | -|------|-------------------|------------------| -| **数据准确性** | 高（从 etcd 读取） | 中（依赖历史记录） | -| **实现复杂度** | 低 | 高 | -| **内存开销** | 低 | 高（需要存储历史） | -| **性能** | 中（需要网络 I/O） | 高（内存操作） | -| **容错性** | 高（可以从 etcd 恢复） | 低（历史可能丢失） | -| **适用场景** | 乱序不频繁、数据准确性要求高 | 乱序频繁、性能要求高 | - -## 推荐方案 - -**推荐使用方案 A（基于 etcd 的完整重放）**，原因： - -1. **数据准确性高**：从 etcd 读取保证数据正确 -2. **实现简单**：不需要维护操作历史 -3. **内存友好**：不需要额外存储 -4. **容错性好**：即使本地状态丢失也能恢复 - -**优化建议**： -1. **异步回滚**：不阻塞正常处理 -2. **批量读取**：减少网络往返 -3. **限制范围**：只回滚最近的操作 -4. **监控告警**：记录乱序频率，超过阈值时触发全量同步 - -## 测试场景 - -### 1. 正常乱序检测和回滚 - -``` -1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5 -2. 应用成功 -3. Standby 收到 OpLog: sequence_id=102, key="obj1", key_sequence_id=6 -4. 应用成功 -5. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=5 -6. 检测到乱序，触发回滚 -7. 从 etcd 重新读取 obj1 的所有 OpLog -8. 按顺序重新应用 -9. 验证 metadata 正确 -``` - -### 2. 并发回滚保护 - -``` -1. 检测到 key="obj1" 乱序，开始回滚 -2. 回滚期间，收到新的 OpLog: key="obj1" -3. 检测到正在回滚，跳过新 OpLog -4. 回滚完成后，新的 OpLog 可以正常处理 -``` - -### 3. 回滚失败处理 - -``` -1. 检测到乱序，触发回滚 -2. 从 etcd 读取 OpLog 失败 -3. 记录错误日志 -4. 清除回滚标记 -5. 可选：触发全量同步 -``` - -### 4. 频繁乱序处理 - -``` -1. 某个 key 频繁乱序（超过阈值） -2. 记录告警 -3. 触发全量同步 -4. 避免频繁回滚影响性能 -``` - -## 总结 - -### 核心方案 - -**对于乱序的 key，执行回滚和重放**： -1. 回滚：删除该 key 的 metadata -2. 重放：从 etcd 重新读取该 key 的所有 OpLog -3. 重写：按正确顺序重新应用所有 OpLog - -### 关键实现 - -1. **回滚起点**：维护 `key_first_sequence_id_` 记录首次 sequence_id -2. **并发保护**：使用 `keys_under_rollback_` 防止并发回滚 -3. **异步执行**：回滚在后台线程执行，不阻塞正常处理 -4. **批量读取**：从 etcd 批量读取 OpLog，减少网络往返 -5. **监控告警**：记录乱序频率，超过阈值时触发全量同步 - -### 优势 - -1. **数据准确性**：从 etcd 读取保证数据正确 -2. **局部修复**：只影响乱序的 key，不影响其他 key -3. **自动恢复**：自动检测和修复数据不一致 -4. **性能友好**：异步执行，不阻塞正常处理 - -### 注意事项 - -1. **性能影响**：回滚和重放可能耗时，需要异步执行 -2. **并发处理**：回滚期间需要防止并发处理该 key -3. **范围限制**：可以限制回滚范围，只回滚最近的操作 -4. **监控告警**：需要监控乱序频率，超过阈值时考虑全量同步 - diff --git a/doc/zh/rfc-oplog-via-etcd-complete-design.md b/doc/zh/rfc-oplog-via-etcd-complete-design.md deleted file mode 100644 index 58060f3098..0000000000 --- a/doc/zh/rfc-oplog-via-etcd-complete-design.md +++ /dev/null @@ -1,738 +0,0 @@ -# 基于 etcd 的 OpLog 主备同步完整方案 - -## 方案概述 - -使用 etcd 作为中间可靠性组件，实现 Primary Master 和 Standby Master 之间的 OpLog 同步。OpLog 只记录 PUT 和 DELETE 事件，通过 etcd 的 Watch 机制实现实时同步。 - -## 核心设计原则 - -1. **OpLog 只记录 PUT 和 DELETE 事件**：不记录 LEASE_RENEW，减少 OpLog 大小 -2. **etcd 作为中间存储**：利用 etcd 的强一致性和 Watch 机制 -3. **快照集成**：与现有快照机制集成，快照后可以清理旧的 OpLog -4. **时序保证**：通过 sequence_id 和 key 级别的版本控制保证时序 - -## 架构设计 - -``` -┌─────────────────────────────────────────────────────────┐ -│ Primary Master │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ │ -│ │ MasterService│ │ OpLogManager │ │ -│ │ │ │ │ │ -│ │ PutEnd() │─────▶│ Append() │ │ -│ │ Remove() │ │ │ │ -│ │ Eviction │ └──────────────┘ │ -│ └──────────────┘ │ │ -│ │ │ -│ ▼ │ -│ ┌──────────────┐ │ -│ │ EtcdOpLogStore│ │ -│ │ │ │ -│ │ WriteOpLog() │ │ -│ └──────────────┘ │ -│ │ │ -│ │ 写入 etcd │ -│ ▼ │ -│ ┌──────────────┐ │ -│ │ etcd │ │ -│ │ │ │ -│ │ /oplog/{seq} │ │ -│ └──────────────┘ │ -└─────────────────────────────────────────────────────────┘ - │ - │ Watch - ▼ -┌─────────────────────────────────────────────────────────┐ -│ Standby Masters │ -│ │ -│ ┌──────────────────────────────────────────────────┐ │ -│ │ MasterServiceSupervisor │ │ -│ │ - 检测 leader │ │ -│ │ - 启动/停止 HotStandbyService │ │ -│ └──────────────────────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌──────────────┐ ┌──────────────┐ │ -│ │ OpLogWatcher │ │ OpLogApplier │ │ -│ │ │ │ │ │ -│ │ WatchEtcd() │─────▶│ ApplyOpLog() │ │ -│ │ │ │ │ │ -│ └──────────────┘ └──────────────┘ │ -│ │ │ │ -│ │ ▼ │ -│ │ ┌──────────────┐ │ -│ │ │ MetadataStore│ │ -│ │ │ │ │ -│ │ │ 更新 metadata │ │ -│ │ └──────────────┘ │ -│ │ │ -│ └──────────────────────────────────────────────┘ -└─────────────────────────────────────────────────────────┘ -``` - -## etcd Key 设计 - -### 1. OpLog Entry Key - -``` -mooncake-store/oplog/{cluster_id}/{sequence_id} -``` - -**示例**： -``` -mooncake-store/oplog/mooncake_cluster/1 -mooncake-store/oplog/mooncake_cluster/2 -mooncake-store/oplog/mooncake_cluster/3 -... -``` - -**设计考虑**： -- 使用 `sequence_id` 作为 key 的一部分，保证顺序 -- 支持按 sequence_id 范围查询 -- 易于清理（删除指定 sequence_id 之前的 key） - -### 2. 最新 Sequence ID Key - -``` -mooncake-store/oplog/{cluster_id}/latest -``` - -**用途**： -- 存储当前最新的 sequence_id -- Standby 可以快速获取最新的 sequence_id -- 用于快照时记录 OpLog 的 sequence_id - -### 3. 快照 Sequence ID Key - -``` -mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id -``` - -**用途**： -- 记录每个快照对应的 sequence_id -- 用于确定可以清理的 OpLog 范围 - -## OpLog Entry 数据结构 - -```cpp -struct OpLogEntry { - uint64_t sequence_id{0}; // 全局递增序列号 - uint64_t timestamp_ms{0}; // 时间戳（毫秒） - OpType op_type{OpType::PUT_END}; // PUT_END, PUT_REVOKE, REMOVE - std::string object_key; // 对象 key - std::string payload; // 可选负载（用于 PUT_END 时携带 replica 信息） - uint32_t checksum{0}; // 校验和 - uint32_t prefix_hash{0}; // key 前缀哈希 - uint64_t key_sequence_id{0}; // 该 key 的操作序列号（用于时序保证） -}; -``` - -**JSON 序列化格式**： -```json -{ - "sequence_id": 12345, - "timestamp": 1704110400123, - "op_type": "PUT_END", - "key": "object_key_123", - "payload": "optional_payload", - "checksum": 1234567890, - "prefix_hash": 987654321, - "key_sequence_id": 5 -} -``` - -## Primary 端实现 - -### 1. EtcdOpLogStore 类 - -```cpp -class EtcdOpLogStore { -public: - EtcdOpLogStore(const std::string& etcd_endpoints, - const std::string& cluster_id); - - // 写入 OpLog 到 etcd - bool WriteOpLog(const OpLogEntry& entry); - - // 批量写入 OpLog（可选优化） - bool WriteOpLogBatch(const std::vector& entries); - - // 更新最新的 sequence_id - bool UpdateLatestSequenceId(uint64_t sequence_id); - - // 记录快照对应的 sequence_id - bool RecordSnapshotSequenceId(const std::string& snapshot_id, - uint64_t sequence_id); - - // 清理指定 sequence_id 之前的 OpLog - bool CleanupOpLogBefore(uint64_t sequence_id); - -private: - std::string BuildOpLogKey(uint64_t sequence_id); - std::string SerializeOpLogEntry(const OpLogEntry& entry); - OpLogEntry DeserializeOpLogEntry(const std::string& data); - - std::string etcd_prefix_; - std::string cluster_id_; - // etcd client -}; -``` - -### 2. 集成到 OpLogManager - -```cpp -class OpLogManager { -public: - // 设置 EtcdOpLogStore（可选，如果不设置则只写入内存） - void SetEtcdOpLogStore(EtcdOpLogStore* store); - - uint64_t Append(OpType type, const std::string& key, - const std::string& payload = std::string()) { - OpLogEntry entry; - // ... 填充 entry ... - - // 写入内存 buffer - buffer_.emplace_back(entry); - - // 写入 etcd（如果设置了） - if (etcd_store_) { - etcd_store_->WriteOpLog(entry); - etcd_store_->UpdateLatestSequenceId(entry.sequence_id); - } - - return entry.sequence_id; - } - -private: - EtcdOpLogStore* etcd_store_{nullptr}; - // ... 其他成员 ... -}; -``` - -### 3. 驱逐时记录 DELETE 事件 - -```cpp -void MasterService::BatchEvict(...) { - // ... 驱逐逻辑 ... - - if (it->second.lease_timeout <= target_timeout) { - std::string evicted_key = it->first; - - // 驱逐对象 - total_freed_size += it->second.size * it->second.GetMemReplicaCount(); - it->second.EraseReplica(ReplicaType::MEMORY); - - if (it->second.IsValid() == false) { - // 对象完全无效，记录 DELETE 事件 - AppendOpLogAndNotify(OpType::REMOVE, evicted_key); - it = shard.metadata.erase(it); - } else { - ++it; - } - } -} -``` - -## Standby 端实现 - -### 0. Standby 服务集成 - -**问题**：现有代码中，Standby 在等待 leader 选举期间只是阻塞等待，没有运行 Standby 服务来同步 OpLog。 - -**解决方案**：在 Standby 模式下并行运行 Standby 服务，watch etcd OpLog 并实时恢复 metadata。 - -**核心流程**： -1. `MasterServiceSupervisor` 检测到有 leader 时，启动 `HotStandbyService` -2. `HotStandbyService` 启动 `OpLogWatcher` watch etcd OpLog -3. 实时应用 OpLog 到本地 metadata store -4. 选举成功后，停止 Standby 服务并提升为 Primary - -**详细设计请参考**：`doc/zh/rfc-standby-service-integration.md` - -### 1. OpLogWatcher 类 - -```cpp -class OpLogWatcher { -public: - OpLogWatcher(const std::string& etcd_endpoints, - const std::string& cluster_id, - OpLogApplier* applier); - - // 启动 Watch - void Start(); - - // 停止 Watch - void Stop(); - - // 从指定 sequence_id 开始读取历史 OpLog - bool ReadOpLogSince(uint64_t start_seq_id, - std::vector& entries); - -private: - // Watch etcd OpLog 变化 - void WatchOpLog(); - - // 处理 Watch 事件 - void HandleWatchEvent(const WatchEvent& event); - - std::string etcd_prefix_; - std::string cluster_id_; - OpLogApplier* applier_; - std::atomic running_{false}; - std::thread watch_thread_; - uint64_t last_processed_sequence_id_{0}; -}; -``` - -### 2. OpLogApplier 类（时序保证） - -```cpp -class OpLogApplier { -public: - OpLogApplier(MetadataStore* metadata_store); - - // 应用 OpLog Entry（带时序检查） - bool ApplyOpLogEntry(const OpLogEntry& entry); - - // 获取 key 的当前 sequence_id - uint64_t GetKeySequenceId(const std::string& key) const; - - // 恢复处理状态 - void Recover(uint64_t last_applied_sequence_id); - -private: - // 检查时序性 - bool CheckSequenceOrder(const OpLogEntry& entry); - - // 应用 PUT_END - void ApplyPutEnd(const OpLogEntry& entry); - - // 应用 PUT_REVOKE - void ApplyPutRevoke(const OpLogEntry& entry); - - // 应用 REMOVE - void ApplyRemove(const OpLogEntry& entry); - - MetadataStore* metadata_store_; - - // 记录每个 key 的最后 sequence_id（用于时序检查） - std::unordered_map key_sequence_map_; - std::mutex key_sequence_mutex_; - - // 记录待处理的条目（用于处理序列号不连续的情况） - std::map pending_entries_; - uint64_t expected_sequence_id_{1}; - std::mutex pending_mutex_; -}; -``` - -### 3. 时序保证机制 - -```cpp -bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { - // 1. 检查全局序列号连续性 - if (entry.sequence_id != expected_sequence_id_) { - if (entry.sequence_id > expected_sequence_id_) { - // 序列号不连续，缓存待处理 - std::lock_guard lock(pending_mutex_); - pending_entries_[entry.sequence_id] = entry; - - // 等待一段时间，看是否有缺失的条目到达 - ScheduleWaitForMissingEntries(entry.sequence_id); - return false; - } else { - // 序列号小于期望值（可能是重复或乱序） - LOG(WARNING) << "Received out-of-order OpLog entry: " - << "expected=" << expected_sequence_id_ - << ", received=" << entry.sequence_id; - return false; - } - } - - // 2. 检查 key 级别的时序性 - if (!CheckSequenceOrder(entry)) { - LOG(ERROR) << "Key-level sequence order violation for key: " - << entry.object_key - << ", entry_seq=" << entry.key_sequence_id - << ", current_seq=" << GetKeySequenceId(entry.object_key); - - // 触发回滚和重放（异步执行） - // 详细设计请参考：doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md - RollbackAndReplayKey(entry.object_key); - return false; - } - - // 3. 应用 OpLog - switch (entry.op_type) { - case OpType::PUT_END: - ApplyPutEnd(entry); - break; - case OpType::PUT_REVOKE: - ApplyPutRevoke(entry); - break; - case OpType::REMOVE: - ApplyRemove(entry); - break; - default: - LOG(WARNING) << "Unknown OpType: " - << static_cast(entry.op_type); - return false; - } - - // 4. 更新状态 - { - std::lock_guard lock(key_sequence_mutex_); - key_sequence_map_[entry.object_key] = entry.key_sequence_id; - } - - expected_sequence_id_++; - - // 5. 处理待处理的条目 - ProcessPendingEntries(); - - return true; -} - -bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { - std::lock_guard lock(key_sequence_mutex_); - - auto it = key_sequence_map_.find(entry.object_key); - if (it == key_sequence_map_.end()) { - // 新 key，允许 - return true; - } - - // 检查 key_sequence_id 是否递增 - if (entry.key_sequence_id <= it->second) { - // 序列号乱序，需要回滚和重放 - return false; - } - - return true; -} -``` - -### 4. 初始同步流程 - -```cpp -class StandbyInitialSync { -public: - void PerformInitialSync() { - // Step 1: 从 Primary 获取快照 - MetadataSnapshot snapshot = RequestSnapshotFromPrimary(); - - // Step 2: 获取快照对应的 sequence_id - uint64_t snapshot_seq_id = snapshot.last_oplog_sequence_id; - - // Step 3: 应用快照 - metadata_store_->ImportSnapshot(snapshot); - - // Step 4: 从 etcd 读取快照后的 OpLog - std::vector entries; - op_log_watcher_->ReadOpLogSince(snapshot_seq_id + 1, entries); - - // Step 5: 应用历史 OpLog - for (const auto& entry : entries) { - applier_->ApplyOpLogEntry(entry); - } - - // Step 6: 开始 Watch 增量 OpLog - op_log_watcher_->Start(); - } -}; -``` - -## 快照集成 - -### 1. 快照时记录 Sequence ID - -```cpp -class SnapshotManager { -public: - MetadataSnapshot CreateSnapshot() { - MetadataSnapshot snapshot; - - // 1. 导出 metadata - snapshot.metadata = ExportMetadata(); - - // 2. 记录当前的 OpLog sequence_id - snapshot.last_oplog_sequence_id = oplog_manager_->GetLastSequenceId(); - - // 3. 将快照信息写入 etcd - std::string snapshot_id = GenerateSnapshotId(); - etcd_oplog_store_->RecordSnapshotSequenceId( - snapshot_id, snapshot.last_oplog_sequence_id); - - // 4. 清理旧的 OpLog - etcd_oplog_store_->CleanupOpLogBefore( - snapshot.last_oplog_sequence_id); - - return snapshot; - } -}; -``` - -### 2. OpLog 清理策略 - -**方案：从 etcd 查询最小的 sequence_id，然后使用 DeleteRange 删除** - -```cpp -bool EtcdOpLogStore::CleanupOpLogBefore(uint64_t target_sequence_id) { - if (target_sequence_id <= 1) { - return true; // 没有需要清理的 - } - - // 1. 从 etcd 查询最小的 sequence_id - uint64_t min_seq_id = GetMinSequenceId(); - - // 2. 如果 min_seq_id >= target_sequence_id，无需清理 - if (min_seq_id >= target_sequence_id) { - return true; - } - - // 3. 执行 DeleteRange - std::string start_key = BuildOpLogKey(min_seq_id); - std::string end_key = BuildOpLogKey(target_sequence_id); - - int64_t deleted_count = 0; - auto err = EtcdHelper::DeleteRange( - start_key.c_str(), start_key.size(), - end_key.c_str(), end_key.size(), - deleted_count); - - if (err != ErrorCode::OK) { - LOG(ERROR) << "Failed to cleanup OpLog"; - return false; - } - - LOG(INFO) << "Cleaned up " << deleted_count - << " OpLog entries from " << min_seq_id - << " to " << target_sequence_id; - return true; -} - -uint64_t EtcdOpLogStore::GetMinSequenceId() const { - // 从 etcd 查询最小的 OpLog sequence_id - std::string prefix = etcd_prefix_ + "/" + cluster_id_ + "/"; - std::string first_key, first_value; - - auto err = EtcdHelper::GetFirstKeyWithPrefix( - prefix.c_str(), prefix.size(), - first_key, first_value); - - if (err == ErrorCode::OK) { - // 从 key 中提取 sequence_id - uint64_t min_seq_id = ExtractSequenceIdFromKey(first_key); - if (min_seq_id > 0) { - return min_seq_id; - } - } - - // Fallback：从快照记录获取 - uint64_t last_snapshot_seq_id = GetLastSnapshotSequenceId(); - if (last_snapshot_seq_id > 0) { - return last_snapshot_seq_id; - } - - // 保守策略：从 1 开始 - return 1; -} -``` - -**详细实现请参考：`doc/zh/rfc-oplog-cleanup-start-sequence-id.md`** - -## 时序保证机制详解 - -### 1. 全局序列号（sequence_id） - -- **作用**：保证所有 OpLog 事件的全局顺序 -- **生成**：Primary 端 OpLogManager 全局递增 -- **检查**：Standby 端检查 sequence_id 是否连续 - -### 2. Key 级别序列号（key_sequence_id） - -- **作用**：保证同一个 key 的操作顺序 -- **生成**：Primary 端为每个 key 维护独立的序列号 -- **检查**：Standby 端检查 key_sequence_id 是否递增 - -### 3. 序列号不连续处理 - -```cpp -void OpLogApplier::ScheduleWaitForMissingEntries(uint64_t missing_seq) { - // 等待一段时间（如 1 秒） - std::this_thread::sleep_for(std::chrono::seconds(1)); - - // 如果缺失的条目仍未到达，需要从 etcd 读取 - if (pending_entries_.find(missing_seq) == pending_entries_.end()) { - RequestMissingOpLog(missing_seq); - } -} - -void OpLogApplier::RequestMissingOpLog(uint64_t missing_seq) { - // 从 etcd 读取缺失的 OpLog - OpLogEntry entry; - if (ReadOpLogFromEtcd(missing_seq, entry)) { - ApplyOpLogEntry(entry); - } else { - LOG(ERROR) << "Failed to read missing OpLog: seq=" << missing_seq; - // 触发重新同步 - TriggerResync(); - } -} -``` - -## 实现步骤 - -### Phase 1：基础框架（优先级：高） - -1. **实现 EtcdOpLogStore** - - 写入 OpLog 到 etcd - - 更新最新 sequence_id - - 读取 OpLog 从 etcd - -2. **集成到 OpLogManager** - - 添加 EtcdOpLogStore 成员 - - 在 Append 时写入 etcd - -3. **实现 OpLogWatcher** - - Watch etcd OpLog 变化 - - 处理 Watch 事件 - -### Phase 2：Standby 端处理（优先级：高） - -1. **实现 OpLogApplier** - - 应用 OpLog Entry - - 时序检查逻辑 - - 处理序列号不连续 - -2. **实现初始同步** - - 从 Primary 获取快照 - - 读取历史 OpLog - - 应用快照和 OpLog - -### Phase 3：快照集成（优先级：中） - -1. **快照时记录 sequence_id** - - 在快照中记录 last_oplog_sequence_id - - 写入 etcd - -2. **OpLog 清理** - - 实现 CleanupOpLogBefore - - 定期清理旧的 OpLog - -### Phase 4：优化（优先级：低） - -1. **批量写入** - - 实现 WriteOpLogBatch - - 减少 etcd 写入次数 - -2. **压缩** - - OpLog Entry 压缩 - - 减少 etcd 存储大小 - -## 关键设计要点 - -### 1. etcd Key 设计 - -- 使用顺序 Key：`mooncake-store/oplog/{cluster_id}/{sequence_id}` -- 支持按 sequence_id 范围查询 -- 易于清理（删除指定 sequence_id 之前的 key） - -### 2. 时序保证 - -- **全局序列号**：保证所有事件的全局顺序 -- **Key 级别序列号**：保证同一 key 的操作顺序 -- **序列号不连续处理**：检测并处理序列号不连续的情况 -- **序列号乱序处理**：检测到 key 级别乱序时，执行回滚和重放（详细设计请参考：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md`） - -### 3. 快照集成 - -- 快照时记录 sequence_id -- 快照后清理旧的 OpLog -- Standby 从快照点开始应用增量 OpLog - -### 4. 故障恢复 - -- Standby 持久化处理状态 -- 支持断点续传 -- 发现不一致时触发重新同步 - -### 5. Standby 服务集成 - -**问题**：现有代码中，Standby 在等待 leader 选举期间只是阻塞等待，没有运行 Standby 服务来同步 OpLog。 - -**解决方案**：在 Standby 模式下并行运行 Standby 服务，watch etcd OpLog 并实时恢复 metadata。 - -**详细设计请参考**：`doc/zh/rfc-standby-service-integration.md` - -### 6. Standby 提升为 Primary 时的 Lease 初始化 - -**问题**：Standby 上的对象 lease 都是 0（因为 OpLog 只包含 PUT_END，不包含续约信息），提升为 Primary 后所有对象会立即过期。 - -**解决方案**：在 `Promote()` 时，给所有 lease 为 0 的对象授予默认租约时间（`default_kv_lease_ttl`）。 - -**详细设计请参考**：`doc/zh/rfc-standby-promotion-lease-initialization.md` - -### 7. 序列号乱序时的回滚和重放 - -**问题**：当检测到某个 key 的 `key_sequence_id` 乱序时，该 key 的 metadata 可能已经不一致。 - -**解决方案**：对于乱序的 key，执行回滚和重放： -1. **回滚**：从 metadata_store 中删除该 key 的所有状态 -2. **重放**：从该 key 第一次出现的 sequence_id 开始，从 etcd 重新读取所有 OpLog -3. **重写**：按正确顺序重新应用所有 OpLog，重建 metadata - -**关键设计**： -- 异步执行回滚，不阻塞正常处理 -- 使用 `keys_under_rollback_` 防止并发回滚 -- 从 etcd 批量读取 OpLog，减少网络往返 -- 监控乱序频率，超过阈值时触发全量同步 - -**详细设计请参考**：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md` - -### 8. key_sequence_map_ 内存清理策略 - -**问题**：Standby 端的 `OpLogApplier` 中，`key_sequence_map_` 用于跟踪每个 key 的 `key_sequence_id`。当 metadata 被删除后，这些条目仍然保留用于乱序检测，长期运行可能导致内存泄漏。 - -**解决方案**：实现定期清理机制： -1. **清理条件**：最后一次操作是 `REMOVE` 且距离当前超过 1 小时 -2. **清理频率**：每小时扫描一次 -3. **保留策略**：`PUT_END` 和 `PUT_REVOKE` 操作的 key 不清理（metadata 可能仍存在） - -**关键设计**： -- 在 `ApplyOpLogEntry` 中触发清理检查，无需额外线程 -- 只清理 `REMOVE` 操作且超过 1 小时的条目 -- 1 小时的时间窗口足够处理网络延迟、重传等异常情况 -- 有效控制内存占用，从潜在的 90MB+ 降低到约 `(活跃key数量 + 1万) × 90字节` - -**详细设计请参考**：`doc/zh/rfc-oplog-key-sequence-map-cleanup.md` - -## 与现有方案对比 - -| 特性 | 当前方案（gRPC 推送） | etcd Watch 方案 | -|------|----------------------|----------------| -| **时序保证** | 依赖网络顺序 | etcd 保证顺序 | -| **可靠性** | 需要 ACK 机制 | etcd 保证可靠性 | -| **断点续传** | 需要实现 | etcd 原生支持 | -| **数据持久化** | 需要额外实现 | etcd 自动持久化 | -| **快照集成** | 需要额外实现 | 易于集成 | -| **实现复杂度** | 高 | 中等 | - -## 实施计划 - -详细的实施计划、优先级和时间估算请参考：`doc/zh/rfc-oplog-implementation-plan.md` - -**实施阶段总览**： -- **Phase 1**：基础框架（P0，2-3 周） -- **Phase 2**：Standby 服务集成（P0，2-3 周） -- **Phase 3**：时序保证和容错（P1，2-3 周） -- **Phase 4**：快照集成和清理（P2，1-2 周） -- **Phase 5**：优化和完善（P3，1-2 周） - -**总计**：8-13 周（约 2-3 个月） - -## 总结 - -本方案利用 etcd 的强一致性和 Watch 机制，实现了可靠的 OpLog 同步。通过只记录 PUT 和 DELETE 事件，大幅减少了 OpLog 大小。通过全局和 key 级别的序列号，保证了时序性。通过与快照机制集成，实现了高效的 OpLog 清理。 - diff --git a/doc/zh/rfc-standby-no-response-handling.md b/doc/zh/rfc-standby-no-response-handling.md deleted file mode 100644 index a46ab68d8d..0000000000 --- a/doc/zh/rfc-standby-no-response-handling.md +++ /dev/null @@ -1,355 +0,0 @@ -# Standby Master 无响应处理方案 - -## 问题分析 - -当 OpLog 从 Primary Master 同步到 Standby Master 时，如果 Standby 一直不响应，会导致以下问题： - -### 1. **内存压力** -- `OpLogManager` 的 buffer 有上限（`kMaxBufferEntries_ = 100000`），但即使有上限，也可能导致： - - 内存占用持续增长 - - 无法及时 truncate，导致 buffer 长期占用 - - 如果多个 Standby 都无响应，问题会放大 - -### 2. **数据丢失风险** -- 如果 buffer 满了，最老的 OpLog 会被丢弃（`pop_front()`） -- 如果 Standby 后来恢复，可能无法完整同步历史数据 - -### 3. **性能影响** -- 持续尝试发送失败的消息会消耗 CPU -- 阻塞其他正常 Standby 的同步（如果实现不当） - -### 4. **故障检测缺失** -- 当前实现无法区分： - - **网络分区**：Standby 节点正常，但网络不通 - - **节点故障**：Standby 节点宕机 - - **处理慢**：Standby 节点正常，但处理速度慢 - -## 解决方案设计 - -### 方案 1: 超时检测 + 故障隔离（推荐） - -#### 1.1 添加超时检测机制 - -```cpp -struct StandbyState { - std::shared_ptr stream; - uint64_t acked_seq_id{0}; - std::chrono::steady_clock::time_point last_ack_time; - std::chrono::steady_clock::time_point last_send_time; // 新增 - std::vector pending_batch; - - // 新增：超时和重试状态 - enum class State { - HEALTHY, // 正常状态 - SLOW, // 响应慢，但还在处理 - TIMEOUT, // 超时，可能故障 - DISCONNECTED // 已断开连接 - }; - State state{State::HEALTHY}; - uint32_t consecutive_failures{0}; // 连续失败次数 -}; -``` - -#### 1.2 实现超时检测逻辑 - -```cpp -class ReplicationService { -private: - // 配置参数 - static constexpr uint32_t kAckTimeoutMs = 5000; // ACK 超时时间（5秒） - static constexpr uint32_t kSendTimeoutMs = 3000; // 发送超时时间（3秒） - static constexpr uint32_t kMaxConsecutiveFailures = 3; // 最大连续失败次数 - static constexpr uint32_t kHealthCheckIntervalMs = 1000; // 健康检查间隔（1秒） - - // 定期检查 Standby 健康状态 - void CheckStandbyHealth(); - - // 标记 Standby 为故障状态 - void MarkStandbyUnhealthy(const std::string& standby_id); - - // 尝试恢复 Standby 连接 - void TryRecoverStandby(const std::string& standby_id); -}; -``` - -#### 1.3 故障隔离策略 - -**策略 A: 暂停发送（推荐）** -- 当 Standby 超时或连续失败时，暂停向该 Standby 发送新的 OpLog -- 继续向其他健康的 Standby 发送 -- 保留该 Standby 的 `acked_seq_id`，等待恢复后从断点继续 - -**策略 B: 降级处理** -- 将 Standby 标记为 `SLOW` 状态 -- 降低发送频率（例如：每 10 个 OpLog 发送一次） -- 如果持续超时，再升级为 `TIMEOUT` 状态 - -#### 1.4 实现示例 - -```cpp -void ReplicationService::CheckStandbyHealth() { - std::unique_lock lock(mutex_); - auto now = std::chrono::steady_clock::now(); - - for (auto& [standby_id, state] : standbys_) { - // 检查连接状态 - if (!state.stream || !state.stream->IsConnected()) { - state.state = StandbyState::State::DISCONNECTED; - continue; - } - - // 检查 ACK 超时 - auto ack_age = std::chrono::duration_cast( - now - state.last_ack_time).count(); - - if (ack_age > kAckTimeoutMs) { - state.consecutive_failures++; - - if (state.consecutive_failures >= kMaxConsecutiveFailures) { - state.state = StandbyState::State::TIMEOUT; - LOG(WARNING) << "Standby " << standby_id - << " marked as TIMEOUT after " - << state.consecutive_failures << " failures"; - // 暂停向该 Standby 发送 - } else { - state.state = StandbyState::State::SLOW; - LOG(WARNING) << "Standby " << standby_id - << " is slow (ack_age=" << ack_age << "ms)"; - } - } else { - // 恢复正常 - if (state.state != StandbyState::State::HEALTHY) { - LOG(INFO) << "Standby " << standby_id << " recovered"; - state.state = StandbyState::State::HEALTHY; - state.consecutive_failures = 0; - } - } - } -} - -void ReplicationService::BroadcastEntry(const OpLogEntry& entry) { - std::shared_lock lock(mutex_); - - for (auto& [standby_id, state] : standbys_) { - // 跳过故障的 Standby - if (state.state == StandbyState::State::TIMEOUT || - state.state == StandbyState::State::DISCONNECTED) { - continue; - } - - state.pending_batch.push_back(entry); - - if (state.pending_batch.size() >= kBatchSize) { - SendBatch(standby_id, state.pending_batch); - state.pending_batch.clear(); - } - } -} -``` - -### 方案 2: 真正的 ACK 机制 - -当前实现中，`acked_seq_id` 的更新是假设 `Send()` 成功就更新，这是不正确的。应该： - -1. **发送时记录待确认的序列号** -2. **等待 Standby 的 ACK 响应** -3. **只有收到 ACK 后才更新 `acked_seq_id`** - -```cpp -struct StandbyState { - // ... - std::map pending_acks; // seq_id -> send_time - uint64_t last_sent_seq_id{0}; // 最后发送的序列号 -}; - -void ReplicationService::SendBatch(const std::string& standby_id, - const std::vector& entries) { - // ... 发送逻辑 ... - - if (success && !entries.empty()) { - uint64_t last_seq = entries.back().sequence_id; - state.last_sent_seq_id = last_seq; - // 记录待确认的序列号 - state.pending_acks[last_seq] = std::chrono::steady_clock::now(); - // 注意：这里不更新 acked_seq_id，等收到 ACK 再更新 - } -} - -// 处理 Standby 的 ACK 响应 -void ReplicationService::OnAck(const std::string& standby_id, uint64_t acked_seq_id) { - std::unique_lock lock(mutex_); - auto it = standbys_.find(standby_id); - if (it == standbys_.end()) { - return; - } - - auto& state = it->second; - if (acked_seq_id > state.acked_seq_id) { - state.acked_seq_id = acked_seq_id; - state.last_ack_time = std::chrono::steady_clock::now(); - state.consecutive_failures = 0; // 重置失败计数 - - // 清理已确认的 pending_acks - auto ack_it = state.pending_acks.begin(); - while (ack_it != state.pending_acks.end()) { - if (ack_it->first <= acked_seq_id) { - ack_it = state.pending_acks.erase(ack_it); - } else { - ++ack_it; - } - } - } -} -``` - -### 方案 3: 流控（Backpressure）机制 - -如果 Standby 处理慢，应该限制发送速度，避免 Standby 内存溢出： - -```cpp -struct StandbyState { - // ... - size_t in_flight_bytes{0}; // 正在传输的字节数 - size_t max_in_flight_bytes{10 * 1024 * 1024}; // 最大 10MB - uint32_t pending_batch_count{0}; // 待确认的批次数量 - uint32_t max_pending_batches{10}; // 最大待确认批次 -}; - -bool ReplicationService::CanSendToStandby(const StandbyState& state) const { - // 检查流控条件 - if (state.in_flight_bytes >= state.max_in_flight_bytes) { - return false; // 超过流量限制 - } - if (state.pending_batch_count >= state.max_pending_batches) { - return false; // 超过批次限制 - } - return true; -} -``` - -### 方案 4: OpLog Truncate 策略 - -只有当**所有健康的 Standby** 都 ACK 了某个序列号后，才能安全地 truncate： - -```cpp -uint64_t ReplicationService::GetMinAckedSequenceId() const { - std::shared_lock lock(mutex_); - - if (standbys_.empty()) { - // 没有 Standby，可以 truncate 所有 - return oplog_manager_.GetLastSequenceId(); - } - - uint64_t min_acked = UINT64_MAX; - for (const auto& [standby_id, state] : standbys_) { - // 只考虑健康的 Standby - if (state.state == StandbyState::State::HEALTHY || - state.state == StandbyState::State::SLOW) { - min_acked = std::min(min_acked, state.acked_seq_id); - } - } - - return (min_acked == UINT64_MAX) ? 0 : min_acked; -} - -// 定期调用，清理已确认的 OpLog -void ReplicationService::TruncateOpLog() { - uint64_t min_acked = GetMinAckedSequenceId(); - if (min_acked > 0) { - oplog_manager_.TruncateBefore(min_acked); - } -} -``` - -### 方案 5: 重连和恢复机制 - -当 Standby 恢复后，应该能够从断点继续同步： - -```cpp -void ReplicationService::TryRecoverStandby(const std::string& standby_id) { - std::unique_lock lock(mutex_); - auto it = standbys_.find(standby_id); - if (it == standbys_.end()) { - return; - } - - auto& state = it->second; - - // 检查连接是否恢复 - if (state.stream && state.stream->IsConnected()) { - // 从上次 ACK 的位置开始重新发送 - uint64_t start_seq = state.acked_seq_id + 1; - auto entries = oplog_manager_.GetEntriesSince(start_seq, 1000); - - if (!entries.empty()) { - LOG(INFO) << "Recovering Standby " << standby_id - << " from seq_id=" << start_seq - << ", entries=" << entries.size(); - SendBatch(standby_id, entries); - state.state = StandbyState::State::HEALTHY; - } - } -} -``` - -## 实施优先级 - -### Phase 1: 基础超时检测（必须） -1. 添加 `StandbyState::State` 枚举 -2. 实现 `CheckStandbyHealth()` 定期检查 -3. 在 `BroadcastEntry()` 中跳过故障 Standby -4. 添加配置参数（超时时间、最大失败次数） - -### Phase 2: 真正的 ACK 机制（重要） -1. 修改 `SendBatch()` 不立即更新 `acked_seq_id` -2. 添加 `OnAck()` 方法处理 ACK 响应 -3. 实现 `pending_acks` 跟踪机制 - -### Phase 3: 流控和 Truncate（优化） -1. 实现流控机制 -2. 实现安全的 OpLog truncate -3. 添加监控指标（replication lag、failure rate） - -### Phase 4: 恢复机制（完善） -1. 实现重连检测 -2. 实现断点续传 -3. 添加恢复日志 - -## 配置参数建议 - -```cpp -struct ReplicationConfig { - uint32_t ack_timeout_ms = 5000; // ACK 超时时间 - uint32_t send_timeout_ms = 3000; // 发送超时时间 - uint32_t max_consecutive_failures = 3; // 最大连续失败次数 - uint32_t health_check_interval_ms = 1000; // 健康检查间隔 - size_t max_in_flight_bytes = 10 * 1024 * 1024; // 最大传输字节数 - uint32_t max_pending_batches = 10; // 最大待确认批次 - bool enable_backpressure = true; // 是否启用流控 -}; -``` - -## 监控指标 - -建议添加以下监控指标： - -1. **Replication Lag**: 每个 Standby 的延迟（`primary_seq_id - acked_seq_id`） -2. **Failure Rate**: Standby 的失败率 -3. **Timeout Count**: 超时次数 -4. **Recovery Count**: 恢复次数 -5. **OpLog Buffer Size**: OpLog buffer 当前大小 -6. **Truncate Rate**: OpLog truncate 频率 - -## 总结 - -Standby 无响应是一个复杂的分布式系统问题，需要多层次的解决方案： - -1. **超时检测**：及时发现故障 -2. **故障隔离**：避免影响其他 Standby -3. **真正的 ACK**：准确跟踪同步进度 -4. **流控**：保护 Standby 不被压垮 -5. **安全 Truncate**：避免数据丢失 -6. **恢复机制**：支持断点续传 - -建议先实施 Phase 1 和 Phase 2，这两个是最关键的。 - diff --git a/doc/zh/rfc-standby-promotion-lease-initialization.md b/doc/zh/rfc-standby-promotion-lease-initialization.md deleted file mode 100644 index b10daa8921..0000000000 --- a/doc/zh/rfc-standby-promotion-lease-initialization.md +++ /dev/null @@ -1,439 +0,0 @@ -# Standby 提升为 Primary 时的 Lease 初始化方案 - -## 问题描述 - -当 Standby Master 被提升为 Primary Master 时，存在一个关键问题：**所有对象的 lease 都是 0（已过期）**。 - -### 问题根源 - -1. **OpLog 中只包含 PUT 和 DELETE 事件** - - `PUT_END` 事件：在 Primary 上创建对象时，`lease_timeout` 被初始化为 0（立即过期） - - `DELETE` 事件：删除对象 - - **不包含** `LEASE_RENEW` 事件（已从 OpLog 中移除） - -2. **Standby 上的对象状态** - - Standby 从 Primary 同步 OpLog，只收到 `PUT_END` 事件 - - 因此 Standby 上所有对象的 `lease_timeout` 都是 0（epoch） - - Standby 不执行驱逐，所以不会检查 lease 是否过期 - -3. **提升为 Primary 后的影响** - - 新 Primary 开始执行驱逐逻辑 - - 由于所有对象的 `lease_timeout` 都是 0，所有对象都会立即被判定为过期 - - 这会导致所有对象被立即驱逐，系统无法正常工作 - -### 问题场景示例 - -``` -时间线： -1. Primary: PutEnd(key="obj1") → lease_timeout = 0 -2. Primary: ExistKey(key="obj1") → lease_timeout = now + 5s (续约) -3. Standby: 同步 PUT_END 事件 → lease_timeout = 0 (没有续约信息) -4. Primary 崩溃 -5. Standby 提升为 Primary -6. 新 Primary: 执行驱逐 → 所有对象 lease_timeout = 0 → 全部被驱逐 ❌ -``` - -## 解决方案 - -### 方案：在提升时给所有对象授予默认租约 - -**核心思路**：当 Standby 被提升为 Primary 时，遍历所有 metadata，给每个对象授予一个默认的租约时间。 - -### 实现设计 - -#### 1. 在 `HotStandbyService::Promote()` 中添加 Lease 初始化逻辑 - -```cpp -std::unique_ptr HotStandbyService::Promote() { - if (!IsReadyForPromotion()) { - LOG(ERROR) << "Standby is not ready for promotion"; - return nullptr; - } - - LOG(INFO) << "Promoting Standby to Primary. Applied seq_id: " - << applied_seq_id_.load(); - - // Stop replication - Stop(); - - // 1. 创建新的 MasterService 实例 - auto master_service = std::make_unique(/* config */); - - // 2. 从 metadata_store_ 恢复 metadata 到新的 MasterService - RestoreMetadataToMasterService(*master_service); - - // 3. 【关键】给所有对象授予默认租约 - InitializeLeasesForAllObjects(*master_service); - - // 4. 执行一次完整的驱逐清理（清理真正过期的对象） - PerformFullEvictionCleanup(*master_service); - - LOG(INFO) << "Standby promoted to Primary successfully"; - return master_service; -} -``` - -#### 2. 实现 `InitializeLeasesForAllObjects()` - -```cpp -void HotStandbyService::InitializeLeasesForAllObjects(MasterService& master_service) { - LOG(INFO) << "Initializing leases for all objects after promotion"; - - uint64_t default_lease_ttl = master_service.GetDefaultLeaseTtl(); - uint64_t default_soft_pin_ttl = master_service.GetDefaultSoftPinTtl(); - - size_t initialized_count = 0; - - // 遍历所有 shard 中的所有 metadata - for (auto& shard : master_service.GetMetadataShards()) { - std::unique_lock lock(shard.mutex); - - for (auto& [key, metadata] : shard.metadata) { - // 检查 lease 是否过期（lease_timeout = 0 表示过期） - if (metadata.IsLeaseExpired()) { - // 授予默认租约 - metadata.GrantLease(default_lease_ttl, default_soft_pin_ttl); - initialized_count++; - - VLOG(2) << "Initialized lease for key: " << key - << ", lease_ttl=" << default_lease_ttl; - } - } - } - - LOG(INFO) << "Initialized leases for " << initialized_count - << " objects after promotion"; -} -``` - -#### 3. 实现 `PerformFullEvictionCleanup()` - -```cpp -void HotStandbyService::PerformFullEvictionCleanup(MasterService& master_service) { - LOG(INFO) << "Performing full eviction cleanup after promotion"; - - // 执行一次完整的驱逐，清理真正过期的对象 - // 注意：此时所有对象的 lease 都已经初始化，只有真正过期的对象才会被驱逐 - master_service.BatchEvict(); - - LOG(INFO) << "Full eviction cleanup completed"; -} -``` - -### 关键设计点 - -#### 1. 默认租约时间的选择 - -**选项 A：使用配置的 `default_kv_lease_ttl`** -- **优点**：简单，与正常操作一致 -- **缺点**：可能给已经很久没有访问的对象也授予租约，导致内存浪费 - -**选项 B：使用较短的租约时间（如 1-2 秒）** -- **优点**：快速淘汰真正不活跃的对象 -- **缺点**：可能误杀活跃对象 - -**推荐：选项 A（使用 `default_kv_lease_ttl`）** - -**理由**： -1. 保守策略，避免误杀活跃对象 -2. 如果对象真的不活跃，会在下次驱逐时被清理 -3. 与正常操作一致，行为可预测 - -#### 2. 何时执行 Lease 初始化 - -**时机**：在 `Promote()` 方法中，在恢复 metadata 之后、开始服务请求之前 - -**流程**： -``` -1. 停止 Standby 的复制循环 -2. 创建新的 MasterService 实例 -3. 恢复 metadata 到新的 MasterService -4. 【关键】初始化所有对象的 lease -5. 执行一次完整的驱逐清理 -6. 开始服务请求 -``` - -#### 3. 与驱逐清理的配合 - -**问题**：如果先初始化 lease，再执行驱逐，那么所有对象都有 lease，不会被驱逐？ - -**解答**： -- 初始化 lease 的目的是**防止误杀活跃对象** -- 驱逐清理的目的是**清理真正过期的对象**（基于 `put_start_time` 等条件） -- 实际上，在 Standby 提升时，所有对象都是"新"的（从 OpLog 恢复），所以应该都保留 -- 如果某些对象在 Primary 崩溃前就已经过期，那么它们应该已经被 Primary 驱逐并产生 DELETE 事件，Standby 上不应该有这些对象 - -**更准确的驱逐逻辑**： -- 在 Standby 提升时，不应该基于 lease 进行驱逐 -- 应该基于其他条件（如 `put_start_time` + `put_start_release_timeout_sec_`）进行清理 -- 或者，在提升时**不执行驱逐**，让正常的驱逐循环来处理 - -**修正后的方案**： - -```cpp -void HotStandbyService::Promote() { - // ... 前面的步骤 ... - - // 3. 给所有对象授予默认租约 - InitializeLeasesForAllObjects(*master_service); - - // 4. 【可选】执行一次清理，但只清理明显无效的对象 - // 注意：不基于 lease 进行驱逐，因为所有对象的 lease 都是 0 - // 可以清理：put_start_time 过期的对象、没有完整 replica 的对象等 - CleanupInvalidObjects(*master_service); - - // 5. 启动 MasterService 的驱逐循环 - // 正常的驱逐循环会基于 lease 和其他条件进行驱逐 -} -``` - -## 实现细节 - -### 1. 在 `MasterService` 中添加辅助方法 - -```cpp -class MasterService { -public: - // 获取默认租约 TTL - uint64_t GetDefaultLeaseTtl() const { return default_kv_lease_ttl_; } - - // 获取默认 Soft Pin TTL - uint64_t GetDefaultSoftPinTtl() const { return default_kv_soft_pin_ttl_; } - - // 获取 metadata shards（用于遍历） - std::vector& GetMetadataShards() { return metadata_shards_; } - - // ... 其他方法 ... -}; -``` - -### 2. 在 `HotStandbyService` 中实现 Lease 初始化 - -```cpp -class HotStandbyService { -private: - void InitializeLeasesForAllObjects(MasterService& master_service); - void CleanupInvalidObjects(MasterService& master_service); - - // ... 其他成员 ... -}; - -void HotStandbyService::InitializeLeasesForAllObjects(MasterService& master_service) { - LOG(INFO) << "Initializing leases for all objects after promotion"; - - uint64_t default_lease_ttl = master_service.GetDefaultLeaseTtl(); - uint64_t default_soft_pin_ttl = master_service.GetDefaultSoftPinTtl(); - - size_t initialized_count = 0; - size_t skipped_count = 0; - - // 遍历所有 shard - for (auto& shard : master_service.GetMetadataShards()) { - std::unique_lock lock(shard.mutex); - - for (auto& [key, metadata] : shard.metadata) { - // 只初始化 lease 为 0 的对象 - if (metadata.IsLeaseExpired()) { - metadata.GrantLease(default_lease_ttl, default_soft_pin_ttl); - initialized_count++; - } else { - // 如果 lease 已经有效，说明可能是从快照恢复的，保留原值 - skipped_count++; - } - } - } - - LOG(INFO) << "Lease initialization completed: " - << initialized_count << " objects initialized, " - << skipped_count << " objects skipped"; -} -``` - -### 3. 清理无效对象（可选） - -```cpp -void HotStandbyService::CleanupInvalidObjects(MasterService& master_service) { - LOG(INFO) << "Cleaning up invalid objects after promotion"; - - size_t cleaned_count = 0; - auto now = std::chrono::steady_clock::now(); - - // 遍历所有 shard - for (auto& shard : master_service.GetMetadataShards()) { - std::unique_lock lock(shard.mutex); - - auto it = shard.metadata.begin(); - while (it != shard.metadata.end()) { - auto& [key, metadata] = *it; - - // 清理条件： - // 1. put_start_time 过期且没有完整 replica - // 2. 所有 replica 都无效 - bool should_cleanup = false; - - if (!metadata.HasCompletedReplicas() && - metadata.put_start_time + - master_service.GetPutStartReleaseTimeout() < now) { - should_cleanup = true; - } else if (!metadata.IsValid()) { - should_cleanup = true; - } - - if (should_cleanup) { - VLOG(1) << "Cleaning up invalid object: " << key; - it = shard.metadata.erase(it); - cleaned_count++; - } else { - ++it; - } - } - } - - LOG(INFO) << "Cleaned up " << cleaned_count << " invalid objects"; -} -``` - -## 边界情况处理 - -### 1. 从快照恢复的场景 - -**场景**：Standby 从快照恢复，快照中可能包含 lease 信息 - -**处理**： -- 如果快照中包含 lease 信息，保留原值 -- 如果快照中 lease 为 0，则初始化 - -**实现**： -```cpp -if (metadata.IsLeaseExpired()) { - // lease 为 0，需要初始化 - metadata.GrantLease(default_lease_ttl, default_soft_pin_ttl); -} else { - // lease 已有效，可能是从快照恢复的，保留原值 - skipped_count++; -} -``` - -### 2. 提升过程中的并发访问 - -**场景**：提升过程中，可能有其他线程访问 metadata - -**处理**： -- 使用 `std::unique_lock` 保护每个 shard -- 提升过程应该是原子的（停止 Standby，创建 Primary） - -### 3. 提升失败的处理 - -**场景**：提升过程中发生错误 - -**处理**： -- 记录错误日志 -- 返回 `nullptr`，表示提升失败 -- Standby 继续运行，等待下次提升机会 - -## 性能考虑 - -### 1. 遍历所有对象的开销 - -**影响**： -- 如果对象数量很大（如 100 万），遍历所有对象可能需要几秒 - -**优化**： -- 使用多线程并行处理不同 shard -- 批量处理，减少锁竞争 - -**实现**： -```cpp -void HotStandbyService::InitializeLeasesForAllObjects(MasterService& master_service) { - auto& shards = master_service.GetMetadataShards(); - - // 并行处理所有 shard - std::vector threads; - for (size_t i = 0; i < shards.size(); ++i) { - threads.emplace_back([&shards, i, &master_service]() { - auto& shard = shards[i]; - std::unique_lock lock(shard.mutex); - - for (auto& [key, metadata] : shard.metadata) { - if (metadata.IsLeaseExpired()) { - metadata.GrantLease( - master_service.GetDefaultLeaseTtl(), - master_service.GetDefaultSoftPinTtl()); - } - } - }); - } - - for (auto& t : threads) { - t.join(); - } -} -``` - -### 2. 提升时间窗口 - -**影响**： -- 提升过程需要时间，期间系统不可用 - -**优化**： -- 尽量减少提升时间 -- 可以考虑在 Standby 阶段就预先初始化 lease（但这样 Standby 也需要维护 lease） - -## 测试场景 - -### 1. 正常提升场景 - -``` -1. Standby 同步了 1000 个对象的 PUT_END 事件 -2. 所有对象的 lease_timeout = 0 -3. Primary 崩溃 -4. Standby 提升为 Primary -5. 验证：所有对象的 lease_timeout > now -6. 验证：系统可以正常服务请求 -``` - -### 2. 从快照恢复的场景 - -``` -1. Standby 从快照恢复，快照中包含 lease 信息 -2. 部分对象的 lease_timeout > 0（从快照恢复） -3. 部分对象的 lease_timeout = 0（新同步的） -4. Standby 提升为 Primary -5. 验证：lease_timeout = 0 的对象被初始化 -6. 验证：lease_timeout > 0 的对象保留原值 -``` - -### 3. 大量对象的场景 - -``` -1. Standby 同步了 100 万个对象 -2. Standby 提升为 Primary -3. 验证：所有对象的 lease 都被初始化 -4. 验证：提升时间在可接受范围内（< 10 秒） -``` - -## 总结 - -### 核心方案 - -**在 Standby 提升为 Primary 时，给所有 lease 为 0 的对象授予默认租约时间** - -### 关键点 - -1. **时机**：在 `Promote()` 中，恢复 metadata 之后、开始服务之前 -2. **租约时间**：使用 `default_kv_lease_ttl`（保守策略） -3. **清理**：可选，清理明显无效的对象（不基于 lease） -4. **性能**：并行处理多个 shard，减少提升时间 - -### 优势 - -1. **简单可靠**：逻辑清晰，易于实现和测试 -2. **保守策略**：避免误杀活跃对象 -3. **与现有机制兼容**：使用现有的 `GrantLease` 方法 - -### 注意事项 - -1. **提升时间**：如果对象数量很大，提升可能需要几秒 -2. **内存影响**：给所有对象授予租约，可能暂时保留一些不活跃对象 -3. **后续清理**：正常的驱逐循环会在后续清理不活跃对象 - diff --git a/doc/zh/rfc-standby-service-integration.md b/doc/zh/rfc-standby-service-integration.md deleted file mode 100644 index fa16d2cafa..0000000000 --- a/doc/zh/rfc-standby-service-integration.md +++ /dev/null @@ -1,673 +0,0 @@ -# Standby 服务集成方案 - -## 问题描述 - -在现有代码实现中，Standby Master 在 `MasterServiceSupervisor::Start()` 中只是阻塞等待 leader 失效（`WatchUntilDeleted`），没有运行 Standby 服务来同步 OpLog 和恢复 metadata。 - -### 现有代码的问题 - -```cpp -// MasterServiceSupervisor::Start() -mv_helper.ElectLeader(config_.local_hostname, view_version, lease_id); -// 这里会阻塞等待 leader 失效，期间 Standby 什么都不做 -``` - -**问题**： -1. Standby 在等待期间不执行任何操作 -2. 没有 watch etcd 的 OpLog -3. 没有实时恢复 metadata -4. 提升为 Primary 时，metadata 可能不完整 - -### 我们方案的需求 - -根据基于 etcd 的 OpLog 同步方案，Standby 需要： -1. **Watch etcd 的 OpLog**：实时接收 Primary 写入的 OpLog 事件 -2. **实时恢复 metadata**：将 OpLog 应用到本地 metadata store -3. **在等待选举期间持续运行**：即使不是 leader，也要保持数据同步 - -## 解决方案 - -### 核心思路 - -**在 Standby 模式下并行运行 Standby 服务**： -- 检测到有 leader 时，启动 Standby 服务 -- Standby 服务 watch etcd OpLog 并实时恢复 metadata -- 选举成功后，停止 Standby 服务并提升为 Primary - -### 架构设计 - -``` -┌─────────────────────────────────────────────────────────┐ -│ MasterServiceSupervisor::Start() │ -└─────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 1. 检查当前是否有 leader │ -│ - GetMasterView() │ -│ - 如果有 leader 且不是自己 → Standby 模式 │ -│ - 如果没有 leader → 直接选举 │ -└─────────────────────────────────────────────────────────┘ - │ - ├─ 有 leader (Standby 模式) - │ │ - │ ▼ - │ ┌─────────────────────────────────────────────┐ - │ │ 2. 启动 Standby 服务 │ - │ │ - 创建 HotStandbyService │ - │ │ - 启动 ReplicationLoop (watch etcd) │ - │ │ - 启动 VerificationLoop │ - │ └─────────────────────────────────────────────┘ - │ │ - │ ▼ - │ ┌─────────────────────────────────────────────┐ - │ │ 3. 阻塞等待 leader 失效 │ - │ │ - ElectLeader() (WatchUntilDeleted) │ - │ │ - 期间 Standby 服务持续运行 │ - │ └─────────────────────────────────────────────┘ - │ - └─ 没有 leader (直接选举) - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 4. 选举成功 │ -│ - 停止 Standby 服务（如果正在运行） │ -│ - 检查是否准备好提升 │ -│ - 等待 5 秒防止 split-brain │ -└─────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────┐ -│ 5. 提升为 Primary │ -│ - 调用 HotStandbyService::Promote() │ -│ - 初始化所有对象的 lease │ -│ - 创建 WrappedMasterService │ -│ - 启动 RPC 服务器 │ -└─────────────────────────────────────────────────────────┘ -``` - -## 实现设计 - -### 1. 修改 MasterServiceSupervisor::Start() - -```cpp -int MasterServiceSupervisor::Start() { - while (true) { - LOG(INFO) << "Init master service..."; - coro_rpc::coro_rpc_server server( - config_.rpc_thread_num, config_.rpc_port, config_.rpc_address, - config_.rpc_conn_timeout, config_.rpc_enable_tcp_no_delay); - const char* value = std::getenv("MC_RPC_PROTOCOL"); - if (value && std::string_view(value) == "rdma") { - server.init_ibv(); - } - - LOG(INFO) << "Init leader election helper..."; - MasterViewHelper mv_helper; - if (mv_helper.ConnectToEtcd(config_.etcd_endpoints) != ErrorCode::OK) { - LOG(ERROR) << "Failed to connect to etcd endpoints: " - << config_.etcd_endpoints; - return -1; - } - - // 【新增】检查当前是否有 leader - ViewVersionId current_version = 0; - std::string current_master; - auto ret = mv_helper.GetMasterView(current_master, current_version); - - // 【新增】如果有 leader 且不是自己，启动 Standby 服务 - std::unique_ptr standby_service = nullptr; - if (ret == ErrorCode::OK && current_master != config_.local_hostname) { - LOG(INFO) << "Current leader: " << current_master - << ", starting Standby service..."; - - // 创建并启动 Standby 服务 - HotStandbyConfig standby_config; - standby_config.standby_id = config_.local_hostname; - standby_config.primary_address = current_master; - standby_config.etcd_endpoints = config_.etcd_endpoints; - standby_config.cluster_id = config_.cluster_id; - standby_config.enable_verification = true; - standby_config.max_replication_lag_entries = 1000; - - standby_service = std::make_unique(standby_config); - auto err = standby_service->Start(current_master); - if (err != ErrorCode::OK) { - LOG(ERROR) << "Failed to start Standby service: " << err; - standby_service.reset(); - } else { - LOG(INFO) << "Standby service started, watching OpLog from etcd"; - } - } - - // 尝试选举（如果有 leader，会阻塞等待；如果没有，立即选举） - LOG(INFO) << "Trying to elect self as leader..."; - EtcdLeaseId lease_id = 0; - ViewVersionId view_version = 0; - mv_helper.ElectLeader(config_.local_hostname, view_version, lease_id); - - // 【新增】停止 Standby 服务（如果正在运行） - if (standby_service) { - LOG(INFO) << "Stopping Standby service before promotion..."; - standby_service->Stop(); - - // 【新增】检查是否准备好提升 - if (!standby_service->IsReadyForPromotion()) { - LOG(WARNING) << "Standby is not ready for promotion, " - << "lag: " << standby_service->GetSyncStatus().lag_entries - << " entries, but proceeding anyway due to leader election"; - } - } - - // 防止 split-brain - const int waiting_time = ETCD_MASTER_VIEW_LEASE_TTL; - std::this_thread::sleep_for(std::chrono::seconds(waiting_time)); - - LOG(INFO) << "Starting master service as Primary..."; - - // 【新增】如果 Standby 服务存在，使用它来初始化 MasterService - std::unique_ptr promoted_service = nullptr; - if (standby_service) { - promoted_service = standby_service->Promote(); - if (!promoted_service) { - LOG(ERROR) << "Failed to promote Standby to Primary"; - // 继续使用新的 MasterService，但 metadata 可能不完整 - } else { - LOG(INFO) << "Successfully promoted Standby to Primary"; - } - } - - // 创建 WrappedMasterService - // 注意：这里需要将 promoted_service 的 metadata 复制到新的 MasterService - // 或者修改 WrappedMasterService 的构造方式，支持从 promoted_service 初始化 - mooncake::WrappedMasterService wrapped_master_service( - mooncake::WrappedMasterServiceConfig(config_, view_version)); - - // TODO: 如果 promoted_service 存在，需要将其 metadata 复制到 wrapped_master_service - // 这需要修改 WrappedMasterService 或 MasterService 的接口 - - mooncake::RegisterRpcService(server, wrapped_master_service); - - // Start a thread to keep the leader alive - auto keep_leader_thread = - std::thread([&server, &mv_helper, lease_id]() { - mv_helper.KeepLeader(lease_id); - LOG(INFO) << "Trying to stop server..."; - server.stop(); - }); - - async_simple::Future ec = - server.async_start(); - if (ec.hasResult()) { - LOG(ERROR) << "Failed to start master service: " - << ec.result().value(); - auto etcd_err = EtcdHelper::CancelKeepAlive(lease_id); - if (etcd_err != ErrorCode::OK) { - LOG(ERROR) << "Failed to cancel keep leader alive: " - << etcd_err; - } - keep_leader_thread.join(); - return -1; - } - - // Block until the server is stopped - auto server_err = std::move(ec).get(); - LOG(ERROR) << "Master service stopped: " << server_err; - - // If the server is closed due to internal errors, we need to manually - // stop keep leader alive. - auto etcd_err = EtcdHelper::CancelKeepAlive(lease_id); - LOG(INFO) << "Cancel keep leader alive: " << etcd_err; - keep_leader_thread.join(); - } - return 0; -} -``` - -### 2. 修改 HotStandbyService::ReplicationLoop() - -```cpp -void HotStandbyService::ReplicationLoop() { - LOG(INFO) << "Replication loop started"; - - // 【新增】创建 OpLogWatcher（使用 etcd Watch） - OpLogWatcher oplog_watcher( - config_.etcd_endpoints, - config_.cluster_id, - this); // HotStandbyService 作为 OpLogApplier - - // 【新增】从上次处理的 sequence_id 开始读取历史 OpLog - uint64_t start_seq_id = applied_seq_id_.load() + 1; - if (start_seq_id > 1) { - std::vector historical_entries; - if (oplog_watcher.ReadOpLogSince(start_seq_id, historical_entries)) { - LOG(INFO) << "Read " << historical_entries.size() - << " historical OpLog entries from sequence_id " - << start_seq_id; - - // 应用历史 OpLog - for (const auto& entry : historical_entries) { - ApplyOpLogEntry(entry); - } - } else { - LOG(WARNING) << "Failed to read historical OpLog, " - << "may need to perform full snapshot sync"; - } - } - - // 【新增】启动 etcd Watch - oplog_watcher.Start(); - is_connected_.store(true); - LOG(INFO) << "OpLog watcher started, watching etcd for new OpLog entries"; - - while (running_.load()) { - // OpLogWatcher 会在后台线程中处理 Watch 事件 - // 当收到新 OpLog 时，会调用 ApplyOpLogEntry() - - // 定期检查同步状态 - auto status = GetSyncStatus(); - if (status.lag_entries > config_.max_replication_lag_entries) { - LOG(WARNING) << "Replication lag is high: " - << status.lag_entries << " entries"; - } - - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - } - - // 【新增】停止 Watch - oplog_watcher.Stop(); - is_connected_.store(false); - LOG(INFO) << "Replication loop stopped"; -} -``` - -### 3. 实现 OpLogWatcher（基于 etcd Watch） - -```cpp -class OpLogWatcher { -public: - OpLogWatcher(const std::string& etcd_endpoints, - const std::string& cluster_id, - OpLogApplier* applier) - : etcd_endpoints_(etcd_endpoints), - cluster_id_(cluster_id), - applier_(applier), - etcd_oplog_store_(etcd_endpoints, cluster_id) { - etcd_prefix_ = "mooncake-store/oplog/" + cluster_id + "/"; - } - - void Start() { - if (running_.load()) { - LOG(WARNING) << "OpLogWatcher is already running"; - return; - } - - running_.store(true); - watch_thread_ = std::thread(&OpLogWatcher::WatchOpLogThreadFunc, this); - LOG(INFO) << "OpLogWatcher started"; - } - - void Stop() { - if (!running_.load()) { - return; - } - - running_.store(false); - if (watch_thread_.joinable()) { - watch_thread_.join(); - } - LOG(INFO) << "OpLogWatcher stopped"; - } - - bool ReadOpLogSince(uint64_t start_seq_id, - std::vector& entries) { - return etcd_oplog_store_.ReadOpLogSince(start_seq_id, 1000, entries); - } - -private: - void WatchOpLogThreadFunc() { - LOG(INFO) << "OpLog watch thread started"; - - // 从上次处理的 sequence_id 开始 Watch - uint64_t start_seq_id = last_processed_sequence_id_ + 1; - std::string watch_prefix = etcd_prefix_; - - while (running_.load()) { - try { - // 使用 etcd Watch 监听 OpLog 变化 - // 这里需要使用 etcd 的 Watch API - // 假设 EtcdHelper 提供了 WatchWithPrefix 方法 - auto watch_result = EtcdHelper::WatchWithPrefix( - watch_prefix.c_str(), - watch_prefix.size(), - [this](const EtcdWatchEvent& event) { - HandleWatchEvent(event); - }); - - if (!watch_result) { - LOG(ERROR) << "Watch failed, retrying..."; - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - } catch (const std::exception& e) { - LOG(ERROR) << "Exception in watch thread: " << e.what(); - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - } - - LOG(INFO) << "OpLog watch thread stopped"; - } - - void HandleWatchEvent(const EtcdWatchEvent& event) { - if (event.type == EtcdWatchEventType::PUT) { - // 解析 OpLog Entry - OpLogEntry entry; - if (DeserializeOpLogEntry(event.value, entry)) { - // 应用 OpLog - if (applier_->ApplyOpLogEntry(entry)) { - last_processed_sequence_id_ = entry.sequence_id; - VLOG(2) << "Applied OpLog entry: sequence_id=" - << entry.sequence_id - << ", op_type=" << static_cast(entry.op_type) - << ", key=" << entry.object_key; - } else { - LOG(WARNING) << "Failed to apply OpLog entry: sequence_id=" - << entry.sequence_id; - } - } else { - LOG(ERROR) << "Failed to deserialize OpLog entry from key: " - << event.key; - } - } else if (event.type == EtcdWatchEventType::DELETE) { - // OpLog 被清理，记录日志 - VLOG(1) << "OpLog entry deleted: " << event.key; - } - } - - std::string etcd_endpoints_; - std::string cluster_id_; - std::string etcd_prefix_; - OpLogApplier* applier_; - EtcdOpLogStore etcd_oplog_store_; - - std::atomic running_{false}; - std::thread watch_thread_; - std::atomic last_processed_sequence_id_{0}; -}; -``` - -### 4. HotStandbyService 实现 OpLogApplier 接口 - -```cpp -class HotStandbyService : public OpLogApplier { -public: - // 实现 OpLogApplier 接口 - bool ApplyOpLogEntry(const OpLogEntry& entry) override { - std::lock_guard lock(mutex_); - - // 检查时序性 - if (!CheckSequenceOrder(entry)) { - LOG(WARNING) << "Sequence order violation for entry: " - << entry.sequence_id; - return false; - } - - // 应用 OpLog - switch (entry.op_type) { - case OpType::PUT_END: - ApplyPutEnd(entry); - break; - case OpType::PUT_REVOKE: - ApplyPutRevoke(entry); - break; - case OpType::REMOVE: - ApplyRemove(entry); - break; - default: - LOG(WARNING) << "Unknown OpType: " - << static_cast(entry.op_type); - return false; - } - - applied_seq_id_.store(entry.sequence_id); - return true; - } - -private: - void ApplyPutEnd(const OpLogEntry& entry) { - // 从 metadata_store_ 创建或更新 metadata - // 这里需要实现完整的 metadata 恢复逻辑 - if (metadata_store_) { - metadata_store_->entry_count++; - } - } - - void ApplyPutRevoke(const OpLogEntry& entry) { - // 处理 PUT_REVOKE - // ... - } - - void ApplyRemove(const OpLogEntry& entry) { - // 从 metadata_store_ 删除 metadata - if (metadata_store_ && metadata_store_->entry_count > 0) { - metadata_store_->entry_count--; - } - } - - bool CheckSequenceOrder(const OpLogEntry& entry) { - // 检查全局序列号 - if (entry.sequence_id <= applied_seq_id_.load()) { - LOG(WARNING) << "Received out-of-order entry: " - << "expected > " << applied_seq_id_.load() - << ", got " << entry.sequence_id; - return false; - } - - // 检查 key 级别的序列号 - // 这里需要维护 key_sequence_map_ - // ... - - return true; - } -}; -``` - -## 关键设计点 - -### 1. Standby 服务生命周期 - -``` -启动阶段： -1. 检测到有 leader → 创建 HotStandbyService -2. 启动 ReplicationLoop → 读取历史 OpLog → 启动 Watch -3. 启动 VerificationLoop（可选） - -运行阶段： -1. OpLogWatcher 持续 watch etcd -2. 收到新 OpLog → 调用 ApplyOpLogEntry -3. 实时更新 metadata_store_ - -提升阶段： -1. 选举成功 → 停止 Standby 服务 -2. 调用 Promote() → 初始化 lease → 创建 MasterService -3. 启动 Primary 服务 -``` - -### 2. 历史 OpLog 读取 - -**策略**： -- 从 `applied_seq_id_ + 1` 开始读取 -- 如果 `applied_seq_id_` 为 0，说明是首次启动，需要从快照开始 -- 批量读取（每次 1000 条），避免一次性读取过多 - -**实现**： -```cpp -uint64_t start_seq_id = applied_seq_id_.load() + 1; -if (start_seq_id == 1) { - // 首次启动，需要从快照恢复 - // 或者从 sequence_id = 1 开始读取所有历史 -} -std::vector entries; -oplog_watcher.ReadOpLogSince(start_seq_id, entries); -``` - -### 3. etcd Watch 实现 - -**关键点**： -- 使用 `WatchWithPrefix` 监听 OpLog 前缀 -- 处理 Watch 断开和重连 -- 处理序列号不连续的情况 - -**Watch 前缀**： -``` -mooncake-store/oplog/{cluster_id}/ -``` - -### 4. 时序保证 - -**全局序列号**： -- 检查 `entry.sequence_id > applied_seq_id_` -- 如果序列号不连续，缓存待处理 - -**Key 级别序列号**： -- 维护 `key_sequence_map_` 记录每个 key 的最后 sequence_id -- 检查 `entry.key_sequence_id > key_sequence_map_[key]` - -## 与现有方案的集成 - -### 1. 与快照机制集成 - -**场景**：Standby 首次启动或需要全量同步 - -**流程**： -1. 检测到 `applied_seq_id_ == 0` 或 lag 过大 -2. 请求 Primary 的快照 -3. 应用快照 -4. 从快照的 `last_oplog_sequence_id` 开始读取增量 OpLog - -### 2. 与提升机制集成 - -**流程**： -1. 选举成功 -2. 停止 Standby 服务 -3. 检查同步状态（`IsReadyForPromotion()`） -4. 调用 `Promote()` → 初始化 lease -5. 创建 MasterService 并启动 - -### 3. 与 OpLog 清理集成 - -**场景**：OpLog 被清理后，Watch 可能收到 DELETE 事件 - -**处理**： -- 记录警告日志 -- 如果发现大量 OpLog 被删除，可能需要重新同步 - -## 错误处理和容错 - -### 1. Watch 断开 - -**处理**： -- 自动重连 -- 从上次处理的 sequence_id 重新 Watch -- 如果重连失败，记录错误并重试 - -### 2. 序列号不连续 - -**处理**： -- 缓存待处理的条目 -- 等待一段时间看是否有缺失的条目到达 -- 如果超时，请求 Primary 或从 etcd 读取缺失的条目 - -### 3. 应用失败 - -**处理**： -- 记录错误日志 -- 不更新 `applied_seq_id_` -- 继续处理后续条目（但可能影响一致性） - -## 性能考虑 - -### 1. Watch 性能 - -- etcd Watch 是高效的，不会产生大量网络开销 -- 批量处理 Watch 事件，减少锁竞争 - -### 2. 历史 OpLog 读取 - -- 批量读取（每次 1000 条） -- 并行应用（如果支持） - -### 3. Metadata 更新 - -- 使用适当的锁粒度 -- 考虑使用无锁数据结构（如果可能） - -## 测试场景 - -### 1. 正常 Standby 运行 - -``` -1. 启动 Standby，检测到有 leader -2. 启动 Standby 服务 -3. Watch etcd OpLog -4. 实时应用 OpLog 到 metadata -5. 验证 metadata 与 Primary 一致 -``` - -### 2. Standby 提升为 Primary - -``` -1. Standby 正在运行 -2. Primary 失效 -3. Standby 选举成功 -4. 停止 Standby 服务 -5. 提升为 Primary -6. 验证 metadata 完整性 -``` - -### 3. Watch 断开重连 - -``` -1. Standby 正在 Watch -2. etcd 连接断开 -3. 自动重连 -4. 从上次处理的 sequence_id 继续 -5. 验证没有丢失 OpLog -``` - -### 4. 历史 OpLog 读取 - -``` -1. Standby 重启 -2. applied_seq_id_ = 1000 -3. 读取 sequence_id >= 1001 的历史 OpLog -4. 应用历史 OpLog -5. 启动 Watch 监听新 OpLog -``` - -## 总结 - -### 核心方案 - -**在 Standby 模式下并行运行 Standby 服务，watch etcd OpLog 并实时恢复 metadata** - -### 关键实现 - -1. **MasterServiceSupervisor**：检测 leader，启动/停止 Standby 服务 -2. **HotStandbyService**：实现 OpLogApplier 接口，管理 Standby 生命周期 -3. **OpLogWatcher**：watch etcd OpLog，处理 Watch 事件 -4. **时序保证**：全局和 key 级别的序列号检查 - -### 优势 - -1. **实时同步**：Standby 实时接收并应用 OpLog -2. **数据完整性**：提升时 metadata 已完整 -3. **自动恢复**：Watch 断开自动重连 -4. **与现有方案兼容**：不影响现有的选举和提升逻辑 - -### 注意事项 - -1. **Watch 性能**：需要确保 etcd Watch 的性能 -2. **序列号不连续**：需要处理缺失的 OpLog -3. **Metadata 恢复**：需要完整实现 metadata 的恢复逻辑 -4. **提升时的数据迁移**：需要将 Standby 的 metadata 迁移到 Primary - From 1b4d3f1d82b2db0e3e7c322a6c03302d0e1e8e3e Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 5 Jan 2026 11:09:56 +0800 Subject: [PATCH 42/59] fix V2 --- mooncake-store/include/etcd_helper.h | 14 +----- mooncake-store/include/oplog_watcher.h | 33 +++---------- mooncake-store/src/etcd_helper.cpp | 32 ------------- mooncake-store/src/oplog_watcher.cpp | 66 ++++---------------------- 4 files changed, 18 insertions(+), 127 deletions(-) diff --git a/mooncake-store/include/etcd_helper.h b/mooncake-store/include/etcd_helper.h index e9a6044a76..b506c4fa8d 100644 --- a/mooncake-store/include/etcd_helper.h +++ b/mooncake-store/include/etcd_helper.h @@ -178,24 +178,14 @@ class EtcdHelper { /* * @brief Watch all keys with a given prefix from a specific etcd revision. - * This is used to close the "read historical -> start watch" gap. - * @param start_revision: Watch events with revision >= start_revision (0 means from now). - */ - static ErrorCode WatchWithPrefixFromRevision( - const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, - void* callback_context, - void (*callback_func)(void*, const char*, size_t, const char*, size_t, int)); - - /* - * @brief Watch all keys with a given prefix from a specific etcd revision (V2). - * V2 callback includes `mod_revision` for precise resume. + * Callback includes `mod_revision` for precise resume. * (Implementation may pass max(event.ModRevision, watchResp.Header.Revision).) * @param callback_func: void cb(void* ctx, const char* key, size_t key_size, * const char* value, size_t value_size, * int event_type, int64_t mod_revision) * event_type: 0=PUT, 1=DELETE, 2=WATCH_BROKEN (watch ended; reconnect) */ - static ErrorCode WatchWithPrefixFromRevisionV2( + static ErrorCode WatchWithPrefixFromRevision( const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, void* callback_context, void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, diff --git a/mooncake-store/include/oplog_watcher.h b/mooncake-store/include/oplog_watcher.h index f0be684fd2..09d9711297 100644 --- a/mooncake-store/include/oplog_watcher.h +++ b/mooncake-store/include/oplog_watcher.h @@ -52,15 +52,6 @@ class OpLogWatcher { */ void Stop(); - /** - * @brief Read OpLog entries from etcd since a given sequence ID - * @param start_seq_id Starting sequence ID (exclusive) - * @param entries Output vector of OpLog entries - * @return true on success, false on failure - */ - bool ReadOpLogSince(uint64_t start_seq_id, - std::vector& entries); - /** * @brief Get the last processed sequence ID * @return Last processed sequence ID @@ -68,25 +59,13 @@ class OpLogWatcher { uint64_t GetLastProcessedSequenceId() const; private: - bool ReadOpLogSinceWithRevision(uint64_t start_seq_id, - std::vector& entries, - EtcdRevisionId& revision_id); - /** - * @brief Static callback function for etcd Watch - * @param context OpLogWatcher instance (passed as void*) - * @param key etcd key - * @param key_size key size - * @param value etcd value - * @param value_size value size - * @param event_type event type (0 = PUT, 1 = DELETE) - */ + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries, + EtcdRevisionId& revision_id); + // Callback includes etcd KV mod_revision for precise resume. static void WatchCallback(void* context, const char* key, size_t key_size, - const char* value, size_t value_size, int event_type); - - // V2 callback includes etcd KV mod_revision for precise resume. - static void WatchCallbackV2(void* context, const char* key, size_t key_size, - const char* value, size_t value_size, int event_type, - int64_t mod_revision); + const char* value, size_t value_size, int event_type, + int64_t mod_revision); /** * @brief Watch etcd OpLog changes (runs in background thread) diff --git a/mooncake-store/src/etcd_helper.cpp b/mooncake-store/src/etcd_helper.cpp index df34f93945..2f2ee3556b 100644 --- a/mooncake-store/src/etcd_helper.cpp +++ b/mooncake-store/src/etcd_helper.cpp @@ -266,25 +266,6 @@ ErrorCode EtcdHelper::WatchWithPrefix(const char* prefix, const size_t prefix_si } ErrorCode EtcdHelper::WatchWithPrefixFromRevision( - const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, - void* callback_context, - void (*callback_func)(void*, const char*, size_t, const char*, size_t, int)) { - char* err_msg = nullptr; - void* callback_func_ptr = reinterpret_cast(callback_func); - int ret = EtcdStoreWatchWithPrefixFromRevisionWrapper( - (char*)prefix, (int)prefix_size, (GoInt64)start_revision, callback_context, - callback_func_ptr, &err_msg); - if (ret != 0) { - LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) - << ", start_revision=" << (int64_t)start_revision - << ", error=" << err_msg; - free(err_msg); - return ErrorCode::ETCD_OPERATION_ERROR; - } - return ErrorCode::OK; -} - -ErrorCode EtcdHelper::WatchWithPrefixFromRevisionV2( const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, void* callback_context, void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, @@ -419,19 +400,6 @@ ErrorCode EtcdHelper::WatchWithPrefix(const char* prefix, const size_t prefix_si } ErrorCode EtcdHelper::WatchWithPrefixFromRevision( - const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, - void* callback_context, - void (*callback_func)(void*, const char*, size_t, const char*, size_t, int)) { - (void)prefix; - (void)prefix_size; - (void)start_revision; - (void)callback_context; - (void)callback_func; - LOG(FATAL) << "Etcd is not enabled in compilation"; - return ErrorCode::ETCD_OPERATION_ERROR; -} - -ErrorCode EtcdHelper::WatchWithPrefixFromRevisionV2( const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, void* callback_context, void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp index 3fa15fdcb4..7ca475bd59 100644 --- a/mooncake-store/src/oplog_watcher.cpp +++ b/mooncake-store/src/oplog_watcher.cpp @@ -54,7 +54,7 @@ bool OpLogWatcher::StartFromSequenceId(uint64_t start_seq_id) { for (;;) { std::vector batch; EtcdRevisionId rev = 0; - if (!ReadOpLogSinceWithRevision(cursor_seq, batch, rev)) { + if (!ReadOpLogSince(cursor_seq, batch, rev)) { last_read_rev = 0; break; } @@ -116,27 +116,8 @@ void OpLogWatcher::Stop() { } bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, - std::vector& entries) { -#ifdef STORE_USE_ETCD - EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); - ErrorCode err = oplog_store.ReadOpLogSince(start_seq_id, 1000, entries); - if (err != ErrorCode::OK) { - LOG(ERROR) << "Failed to read OpLog since sequence_id=" << start_seq_id - << ", error=" << static_cast(err); - return false; - } - LOG(INFO) << "Read " << entries.size() << " OpLog entries since sequence_id=" - << start_seq_id; - return true; -#else - LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot read OpLog from etcd"; - return false; -#endif -} - -bool OpLogWatcher::ReadOpLogSinceWithRevision(uint64_t start_seq_id, - std::vector& entries, - EtcdRevisionId& revision_id) { + std::vector& entries, + EtcdRevisionId& revision_id) { #ifdef STORE_USE_ETCD EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); ErrorCode err = oplog_store.ReadOpLogSinceWithRevision( @@ -159,28 +140,7 @@ uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { return last_processed_sequence_id_.load(); } -// Static callback function for etcd Watch (defined before WatchOpLog uses it) void OpLogWatcher::WatchCallback(void* context, const char* key, size_t key_size, - const char* value, size_t value_size, int event_type) { - OpLogWatcher* watcher = static_cast(context); - if (watcher == nullptr) { - LOG(ERROR) << "OpLogWatcher context is null"; - return; - } - - std::string key_str; - if (key != nullptr && key_size > 0) { - key_str.assign(key, key_size); - } - std::string value_str; - if (value != nullptr && value_size > 0) { - value_str = std::string(value, value_size); - } - - watcher->HandleWatchEvent(key_str, value_str, event_type, /*mod_revision=*/0); -} - -void OpLogWatcher::WatchCallbackV2(void* context, const char* key, size_t key_size, const char* value, size_t value_size, int event_type, int64_t mod_revision) { OpLogWatcher* watcher = static_cast(context); @@ -210,9 +170,9 @@ void OpLogWatcher::WatchOpLog() { // Start watching - pass static callback function and this pointer as context EtcdRevisionId start_rev = static_cast(next_watch_revision_.load()); - // Always use V2 watcher so we can update next_watch_revision_ precisely. - ErrorCode err = EtcdHelper::WatchWithPrefixFromRevisionV2( - watch_prefix.c_str(), watch_prefix.size(), start_rev, this, WatchCallbackV2); + // Use watcher with mod_revision so we can update next_watch_revision_ precisely. + ErrorCode err = EtcdHelper::WatchWithPrefixFromRevision( + watch_prefix.c_str(), watch_prefix.size(), start_rev, this, WatchCallback); if (err != ErrorCode::OK) { LOG(ERROR) << "Failed to start watch for prefix " << watch_prefix @@ -296,7 +256,7 @@ bool OpLogWatcher::SyncMissedEntries() { std::vector entries; EtcdRevisionId rev = 0; - if (!ReadOpLogSinceWithRevision(last_seq, entries, rev)) { + if (!ReadOpLogSince(last_seq, entries, rev)) { LOG(ERROR) << "Failed to read missed OpLog entries"; return false; } @@ -456,15 +416,9 @@ void OpLogWatcher::Stop() { // No-op when STORE_USE_ETCD is not enabled } -bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, - std::vector& entries) { - LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; - return false; -} - -bool OpLogWatcher::ReadOpLogSinceWithRevision(uint64_t /*start_seq_id*/, - std::vector& /*entries*/, - EtcdRevisionId& /*revision_id*/) { +bool OpLogWatcher::ReadOpLogSince(uint64_t /*start_seq_id*/, + std::vector& /*entries*/, + EtcdRevisionId& /*revision_id*/) { LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; return false; } From bf5af33a492facc651ae1d73467f21b765ee36d5 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 5 Jan 2026 11:41:08 +0800 Subject: [PATCH 43/59] fix --- mooncake-store/include/oplog_applier.h | 2 +- mooncake-store/include/oplog_manager.h | 3 --- mooncake-store/src/etcd_oplog_store.cpp | 2 -- mooncake-store/src/oplog_applier.cpp | 4 +--- mooncake-store/src/oplog_manager.cpp | 6 ------ mooncake-store/src/oplog_watcher.cpp | 1 - 6 files changed, 2 insertions(+), 16 deletions(-) diff --git a/mooncake-store/include/oplog_applier.h b/mooncake-store/include/oplog_applier.h index cbab4623a8..ab15b0fc1b 100644 --- a/mooncake-store/include/oplog_applier.h +++ b/mooncake-store/include/oplog_applier.h @@ -50,7 +50,7 @@ class OpLogApplier { /** * @brief Get the current sequence ID for a key (DEPRECATED) * @param key Object key - * @return Always returns 0 - key_sequence_id is no longer tracked + * @return Always returns 0 - global sequence_id is used for ordering * @deprecated Use global sequence_id for ordering */ uint64_t GetKeySequenceId(const std::string& key) const; diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index 3d9a0504b5..003b12340d 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -39,9 +39,6 @@ struct OpLogEntry { std::string payload; // Serialized extra data (optional) uint32_t checksum{0}; // Checksum of payload (implementation-defined) uint32_t prefix_hash{0}; // Hash of the entire key (for verification and optimization) - // Deprecated: key_sequence_id is kept for backward compatibility only. - // Ordering is guaranteed by global sequence_id. - uint64_t key_sequence_id{0}; }; /** diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp index cc0775200c..b245452313 100644 --- a/mooncake-store/src/etcd_oplog_store.cpp +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -349,7 +349,6 @@ std::string EtcdOpLogStore::SerializeOpLogEntry( root["payload"] = entry.payload; root["checksum"] = static_cast(entry.checksum); root["prefix_hash"] = static_cast(entry.prefix_hash); - root["key_sequence_id"] = static_cast(entry.key_sequence_id); Json::StreamWriterBuilder builder; builder["indentation"] = ""; // Compact format @@ -380,7 +379,6 @@ bool EtcdOpLogStore::DeserializeOpLogEntry(const std::string& json_str, entry.payload = root["payload"].asString(); entry.checksum = root["checksum"].asUInt(); entry.prefix_hash = root["prefix_hash"].asUInt(); - entry.key_sequence_id = root["key_sequence_id"].asUInt64(); } catch (const std::exception& e) { LOG(ERROR) << "Failed to deserialize OpLogEntry: " << e.what(); return false; diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index 2a7e83f690..e6851954c7 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -40,7 +40,7 @@ EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const { } bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { - // Global ordering only (key_sequence_id is deprecated and ignored). + // Global ordering only. // // IMPORTANT: // - Watch callbacks / retries may deliver duplicate or already-applied entries. @@ -139,7 +139,6 @@ size_t OpLogApplier::ApplyOpLogEntries(const std::vector& entries) { } uint64_t OpLogApplier::GetKeySequenceId(const std::string& key) const { - // Deprecated: key_sequence_id is no longer tracked. // Global sequence_id is used for ordering. (void)key; // Suppress unused parameter warning return 0; @@ -370,7 +369,6 @@ OpLogApplier::GapResolveResult OpLogApplier::TryResolveGapsOnceForPromotion( bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { // Only check global sequence order. - // key_sequence_id is no longer used for ordering. return entry.sequence_id == expected_sequence_id_.load(); } diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index a9f5faa5bf..e4ffd85b4c 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -29,11 +29,6 @@ uint64_t OpLogManager::Append(OpType type, const std::string& key, std::unique_lock lock(mutex_); entry.sequence_id = ++last_seq_id_; - - // Note: We use global sequence_id for ordering guarantee. - // key_sequence_id is set to sequence_id for backward compatibility, - // but the actual ordering is based on global sequence_id. - entry.key_sequence_id = entry.sequence_id; if (buffer_.size() >= kMaxBufferEntries_) { buffer_.pop_front(); @@ -72,7 +67,6 @@ OpLogEntry OpLogManager::AllocateEntry(OpType type, const std::string& key, std::unique_lock lock(mutex_); entry.sequence_id = ++last_seq_id_; - entry.key_sequence_id = entry.sequence_id; // deprecated if (buffer_.size() >= kMaxBufferEntries_) { buffer_.pop_front(); diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp index 7ca475bd59..8e01194958 100644 --- a/mooncake-store/src/oplog_watcher.cpp +++ b/mooncake-store/src/oplog_watcher.cpp @@ -383,7 +383,6 @@ bool OpLogWatcher::DeserializeOpLogEntry(const std::string& json_str, entry.payload = root.get("payload", "").asString(); entry.checksum = root.get("checksum", 0).asUInt(); entry.prefix_hash = root.get("prefix_hash", 0).asUInt(); - entry.key_sequence_id = root.get("key_sequence_id", 0).asUInt64(); return true; } From b1221e367a080b389aea412aadbe867a0586bfe9 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 5 Jan 2026 20:18:21 +0800 Subject: [PATCH 44/59] fix important bug --- mooncake-common/etcd/etcd_wrapper.go | 65 +++++++++++++++++++++++ mooncake-store/include/etcd_helper.h | 19 +++++++ mooncake-store/include/etcd_oplog_store.h | 9 ++++ mooncake-store/include/ha_helper.h | 14 ++++- mooncake-store/include/oplog_applier.h | 5 +- mooncake-store/src/client_service.cpp | 23 ++++++++ mooncake-store/src/etcd_helper.cpp | 62 +++++++++++++++++++++ mooncake-store/src/etcd_oplog_store.cpp | 54 +++++++++++++++++-- mooncake-store/src/ha_helper.cpp | 37 ++++++++----- mooncake-store/src/master_service.cpp | 20 ++++++- mooncake-store/src/oplog_applier.cpp | 9 ++-- mooncake-store/src/oplog_watcher.cpp | 6 +-- 12 files changed, 294 insertions(+), 29 deletions(-) diff --git a/mooncake-common/etcd/etcd_wrapper.go b/mooncake-common/etcd/etcd_wrapper.go index ff2f0060fe..ec22d98636 100644 --- a/mooncake-common/etcd/etcd_wrapper.go +++ b/mooncake-common/etcd/etcd_wrapper.go @@ -164,11 +164,14 @@ func NewStoreEtcdClient(endpoints *C.char, errMsg **C.char) int { } endpointStr := C.GoString(endpoints) + // Support multiple endpoints separated by comma or semicolon. + endpointStr = strings.ReplaceAll(endpointStr, ",", ";") endpointList := strings.Split(endpointStr, ";") // Filter out any empty strings that might result from splitting var validEndpoints []string for _, ep := range endpointList { + ep = strings.TrimSpace(ep) if ep != "" { validEndpoints = append(validEndpoints, ep) } @@ -448,6 +451,38 @@ func EtcdStorePutWrapper(key *C.char, keySize C.int, value *C.char, valueSize C. return 0 } +// Create key if absent (CAS on CreateRevision==0). +// Return: +// - 0 on success +// - -2 if key already exists +// - -1 on error +// +//export EtcdStoreCreateWrapper +func EtcdStoreCreateWrapper(key *C.char, keySize C.int, value *C.char, valueSize C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + v := C.GoStringN(value, valueSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + txn := storeClient.Txn(ctx) + resp, err := txn.If(clientv3.Compare(clientv3.CreateRevision(k), "=", 0)). + Then(clientv3.OpPut(k, v)). + Commit() + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + if resp.Succeeded { + return 0 + } + *errMsg = C.CString("key already exists") + return -2 +} + //export EtcdStoreGetWithPrefixWrapper func EtcdStoreGetWithPrefixWrapper(prefix *C.char, prefixSize C.int, keys **C.char, keySizes **C.int, values **C.char, valueSizes **C.int, count *C.int, errMsg **C.char) int { if storeClient == nil { @@ -567,6 +602,36 @@ func EtcdStoreGetFirstKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, fir return 0 } +//export EtcdStoreGetLastKeyWithPrefixWrapper +func EtcdStoreGetLastKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, lastKey **C.char, lastKeySize *C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + resp, err := storeClient.Get( + ctx, p, + clientv3.WithPrefix(), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortDescend), + clientv3.WithLimit(1), + ) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + if len(resp.Kvs) == 0 { + *errMsg = C.CString("no key found with prefix") + return -2 + } + kv := resp.Kvs[0] + *lastKey = C.CString(string(kv.Key)) + *lastKeySize = C.int(len(kv.Key)) + return 0 +} + //export EtcdStoreDeleteRangeWrapper func EtcdStoreDeleteRangeWrapper(startKey *C.char, startKeySize C.int, endKey *C.char, endKeySize C.int, errMsg **C.char) int { if storeClient == nil { diff --git a/mooncake-store/include/etcd_helper.h b/mooncake-store/include/etcd_helper.h index b506c4fa8d..38e48743f8 100644 --- a/mooncake-store/include/etcd_helper.h +++ b/mooncake-store/include/etcd_helper.h @@ -103,6 +103,14 @@ class EtcdHelper { static ErrorCode Put(const char* key, const size_t key_size, const char* value, const size_t value_size); + /* + * @brief Create a key-value pair in etcd if the key does not already exist. + * This is implemented via etcd transaction (CreateRevision == 0). + * @return: OK on success; ETCD_TRANSACTION_FAIL if key already exists. + */ + static ErrorCode Create(const char* key, const size_t key_size, + const char* value, const size_t value_size); + /* * @brief Get all key-value pairs with a given prefix. * @param prefix: The prefix to search for. @@ -145,6 +153,17 @@ class EtcdHelper { const size_t prefix_size, std::string& first_key); + /* + * @brief Get the last key with a given prefix (sorted by key descending). + * @param prefix: The prefix to search for. + * @param prefix_size: The size of the prefix in bytes. + * @param last_key: Output param, the last key found. + * @return: Error code. ETCD_KEY_NOT_EXIST if no key found. + */ + static ErrorCode GetLastKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& last_key); + /* * @brief Delete a range of keys from etcd. * @param start_key: The start key (inclusive). diff --git a/mooncake-store/include/etcd_oplog_store.h b/mooncake-store/include/etcd_oplog_store.h index 55efa78d9d..013c38eea3 100644 --- a/mooncake-store/include/etcd_oplog_store.h +++ b/mooncake-store/include/etcd_oplog_store.h @@ -74,6 +74,12 @@ class EtcdOpLogStore { */ ErrorCode GetLatestSequenceId(uint64_t& sequence_id); + // Stronger (than `/latest`) best-effort query: return the maximum existing + // sequence_id by scanning etcd keys under /oplog/{cluster_id}/ with + // descending key order. + // Return ETCD_KEY_NOT_EXIST if no OpLog exists yet. + ErrorCode GetMaxSequenceId(uint64_t& sequence_id); + /** * @brief Update the latest sequence_id in etcd. * @param sequence_id: The latest sequence_id to update. @@ -138,6 +144,9 @@ class EtcdOpLogStore { // "cleaned_upto" marker. std::optional GetMinSequenceId() const; + // Best-effort: find the maximum existing OpLog sequence_id in etcd. + std::optional GetMaxSequenceIdInternal() const; + /** * @brief Serialize an OpLogEntry to JSON string. * @param entry: The OpLog entry to serialize. diff --git a/mooncake-store/include/ha_helper.h b/mooncake-store/include/ha_helper.h index f6774d6a93..a117cb29ef 100644 --- a/mooncake-store/include/ha_helper.h +++ b/mooncake-store/include/ha_helper.h @@ -26,7 +26,18 @@ class MasterViewHelper { public: MasterViewHelper(const MasterViewHelper&) = delete; MasterViewHelper& operator=(const MasterViewHelper&) = delete; - MasterViewHelper(); + // cluster_id source of truth: + // - If provided, use it. + // - Else fall back to env MC_STORE_CLUSTER_ID. + // - Else fall back to DEFAULT_CLUSTER_ID. + explicit MasterViewHelper(const std::string& cluster_id = std::string()); + + // Update cluster_id (and derived master_view_key_) before using the helper. + // This is mainly for client-side etcd:// usage where cluster_id may be passed + // via connection string. + void SetClusterId(const std::string& cluster_id); + + const std::string& GetMasterViewKey() const { return master_view_key_; } /* * @brief Connect to the etcd cluster. This function should be called at @@ -63,6 +74,7 @@ class MasterViewHelper { ViewVersionId& version); private: + void BuildMasterViewKeyFromClusterId(const std::string& cluster_id); std::string master_view_key_; }; diff --git a/mooncake-store/include/oplog_applier.h b/mooncake-store/include/oplog_applier.h index ab15b0fc1b..993ef81f5c 100644 --- a/mooncake-store/include/oplog_applier.h +++ b/mooncake-store/include/oplog_applier.h @@ -160,8 +160,9 @@ class OpLogApplier { std::atomic expected_sequence_id_{1}; // Constants for missing entry handling - static constexpr int kMissingEntryWaitSeconds = 5; // Wait 5 seconds before requesting - static constexpr int kMissingEntrySkipSeconds = 3; // Wait 3 seconds then skip (avoid global stall) + // IMPORTANT: request must happen BEFORE skip, otherwise we will never request. + static constexpr int kMissingEntryRequestSeconds = 1; // request from etcd after 1s + static constexpr int kMissingEntrySkipSeconds = 3; // skip after 3s (avoid global stall) static constexpr int kMaxPendingEntries = 1000; // Max pending entries before giving up }; diff --git a/mooncake-store/src/client_service.cpp b/mooncake-store/src/client_service.cpp index c62789b634..0193c22e07 100644 --- a/mooncake-store/src/client_service.cpp +++ b/mooncake-store/src/client_service.cpp @@ -175,7 +175,30 @@ tl::expected CheckRegisterMemoryParams(const void* addr, ErrorCode Client::ConnectToMaster(const std::string& master_server_entry) { if (master_server_entry.find("etcd://") == 0) { + // Support optional cluster_id in connection string: + // etcd://?cluster_id= + // If not provided, MasterViewHelper will fall back to env MC_STORE_CLUSTER_ID, + // then DEFAULT_CLUSTER_ID. std::string etcd_entry = master_server_entry.substr(strlen("etcd://")); + std::string cluster_id; + { + const size_t qpos = etcd_entry.find('?'); + if (qpos != std::string::npos) { + const std::string query = etcd_entry.substr(qpos + 1); + etcd_entry = etcd_entry.substr(0, qpos); + const std::string k = "cluster_id="; + const size_t kpos = query.find(k); + if (kpos != std::string::npos) { + const size_t vpos = kpos + k.size(); + size_t vend = query.find('&', vpos); + if (vend == std::string::npos) vend = query.size(); + cluster_id = query.substr(vpos, vend - vpos); + } + } + } + if (!cluster_id.empty()) { + master_view_helper_.SetClusterId(cluster_id); + } // Get master address from etcd auto err = master_view_helper_.ConnectToEtcd(etcd_entry); diff --git a/mooncake-store/src/etcd_helper.cpp b/mooncake-store/src/etcd_helper.cpp index 2f2ee3556b..ae3957a2f9 100644 --- a/mooncake-store/src/etcd_helper.cpp +++ b/mooncake-store/src/etcd_helper.cpp @@ -167,6 +167,24 @@ ErrorCode EtcdHelper::Put(const char* key, const size_t key_size, return ErrorCode::OK; } +ErrorCode EtcdHelper::Create(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + char* err_msg = nullptr; + int ret = EtcdStoreCreateWrapper((char*)key, (int)key_size, (char*)value, + (int)value_size, &err_msg); + if (ret == -2) { + free(err_msg); + return ErrorCode::ETCD_TRANSACTION_FAIL; + } + if (ret != 0) { + LOG(ERROR) << "key=" << std::string(key, key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size, std::vector& keys, std::vector& values) { @@ -227,6 +245,30 @@ ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, return ErrorCode::OK; } +ErrorCode EtcdHelper::GetLastKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& last_key) { + char* err_msg = nullptr; + char* last_key_ptr = nullptr; + int last_key_size = 0; + int ret = EtcdStoreGetLastKeyWithPrefixWrapper((char*)prefix, (int)prefix_size, + &last_key_ptr, &last_key_size, + &err_msg); + if (ret == -2) { + free(err_msg); + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + last_key = std::string(last_key_ptr, last_key_size); + free(last_key_ptr); + return ErrorCode::OK; +} + ErrorCode EtcdHelper::DeleteRange(const char* start_key, const size_t start_key_size, const char* end_key, @@ -352,6 +394,16 @@ ErrorCode EtcdHelper::Put(const char* key, const size_t key_size, return ErrorCode::ETCD_OPERATION_ERROR; } +ErrorCode EtcdHelper::Create(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + (void)key; + (void)key_size; + (void)value; + (void)value_size; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size, std::vector& keys, std::vector& values) { @@ -383,6 +435,16 @@ ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, return ErrorCode::ETCD_OPERATION_ERROR; } +ErrorCode EtcdHelper::GetLastKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& last_key) { + (void)prefix; + (void)prefix_size; + (void)last_key; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + ErrorCode EtcdHelper::DeleteRange(const char* start_key, const size_t start_key_size, const char* end_key, diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp index b245452313..0f5c8222c1 100644 --- a/mooncake-store/src/etcd_oplog_store.cpp +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -53,11 +53,25 @@ ErrorCode EtcdOpLogStore::WriteOpLog(const OpLogEntry& entry) { std::string key = BuildOpLogKey(entry.sequence_id); std::string value = SerializeOpLogEntry(entry); - ErrorCode err = EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), - value.size()); + // Fence: never overwrite an existing OpLog key. + // - If this is a retry of the same entry: key exists with same value => OK. + // - If key exists with different value: conflict => error (signals seq regression / bug). + ErrorCode err = EtcdHelper::Create(key.c_str(), key.size(), value.c_str(), value.size()); + if (err == ErrorCode::ETCD_TRANSACTION_FAIL) { + std::string existing; + EtcdRevisionId rev = 0; + ErrorCode get_err = EtcdHelper::Get(key.c_str(), key.size(), existing, rev); + if (get_err == ErrorCode::OK && existing == value) { + // Idempotent retry. + err = ErrorCode::OK; + } else { + LOG(ERROR) << "OpLog key conflict: seq=" << entry.sequence_id + << ", get_err=" << get_err; + return ErrorCode::ETCD_OPERATION_ERROR; + } + } if (err != ErrorCode::OK) { - LOG(ERROR) << "Failed to write OpLog entry, sequence_id=" - << entry.sequence_id; + LOG(ERROR) << "Failed to write OpLog entry, sequence_id=" << entry.sequence_id; return err; } @@ -230,6 +244,15 @@ ErrorCode EtcdOpLogStore::GetLatestSequenceId(uint64_t& sequence_id) { return ErrorCode::OK; } +ErrorCode EtcdOpLogStore::GetMaxSequenceId(uint64_t& sequence_id) { + auto max_seq_opt = GetMaxSequenceIdInternal(); + if (!max_seq_opt.has_value()) { + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + sequence_id = max_seq_opt.value(); + return ErrorCode::OK; +} + ErrorCode EtcdOpLogStore::UpdateLatestSequenceId(uint64_t sequence_id) { std::string key = BuildLatestKey(); std::string value = std::to_string(sequence_id); @@ -325,6 +348,29 @@ std::optional EtcdOpLogStore::GetMinSequenceId() const { } } +std::optional EtcdOpLogStore::GetMaxSequenceIdInternal() const { + // Entry keys are fixed-width 20-digit numbers, which (in practice) start with '0'. + // Use "/0" to avoid picking up "/latest" which is lexicographically after digits. + std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/0"; + std::string last_key; + ErrorCode err = + EtcdHelper::GetLastKeyWithPrefix(prefix.c_str(), prefix.size(), last_key); + if (err != ErrorCode::OK) { + return std::nullopt; + } + + size_t pos = last_key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= last_key.size()) { + return std::nullopt; + } + std::string seq_str = last_key.substr(pos + 1); + try { + return static_cast(std::stoull(seq_str)); + } catch (...) { + return std::nullopt; + } +} + std::string EtcdOpLogStore::BuildLatestKey() const { std::ostringstream oss; oss << kOpLogPrefix << cluster_id_ << kLatestSuffix; diff --git a/mooncake-store/src/ha_helper.cpp b/mooncake-store/src/ha_helper.cpp index 5a50cb4013..24f1e35271 100644 --- a/mooncake-store/src/ha_helper.cpp +++ b/mooncake-store/src/ha_helper.cpp @@ -11,14 +11,30 @@ namespace mooncake { -MasterViewHelper::MasterViewHelper() { - std::string cluster_id; +namespace { +std::string ResolveClusterIdForMasterView(const std::string& cluster_id) { + if (!cluster_id.empty()) { + return cluster_id; + } const char* cluster_id_env = std::getenv("MC_STORE_CLUSTER_ID"); if (cluster_id_env != nullptr && strlen(cluster_id_env) > 0) { - cluster_id = cluster_id_env; - } else { - cluster_id = "mooncake"; + return std::string(cluster_id_env); } + return DEFAULT_CLUSTER_ID; +} +} // namespace + +MasterViewHelper::MasterViewHelper(const std::string& cluster_id) { + BuildMasterViewKeyFromClusterId(ResolveClusterIdForMasterView(cluster_id)); +} + +void MasterViewHelper::SetClusterId(const std::string& cluster_id) { + BuildMasterViewKeyFromClusterId(ResolveClusterIdForMasterView(cluster_id)); +} + +void MasterViewHelper::BuildMasterViewKeyFromClusterId( + const std::string& cluster_id_in) { + std::string cluster_id = cluster_id_in; // Ensure the cluster_id ends with '/' if not empty if (!cluster_id.empty() && cluster_id.back() != '/') { cluster_id += '/'; @@ -132,7 +148,7 @@ int MasterServiceSupervisor::Start() { } LOG(INFO) << "Init leader election helper..."; - MasterViewHelper mv_helper; + MasterViewHelper mv_helper(config_.cluster_id); if (mv_helper.ConnectToEtcd(config_.etcd_endpoints) != ErrorCode::OK) { LOG(ERROR) << "Failed to connect to etcd endpoints: " << config_.etcd_endpoints; @@ -166,17 +182,10 @@ int MasterServiceSupervisor::Start() { StartStandbyService(mv_helper, current_leader); had_standby = true; - // Build master_view_key (same logic as MasterViewHelper) - std::string cluster_id = config_.cluster_id; - if (!cluster_id.empty() && cluster_id.back() != '/') { - cluster_id += '/'; - } - std::string master_view_key = "mooncake-store/" + cluster_id + "master_view"; - // Watch until leader is deleted LOG(INFO) << "Watching for leadership change..."; auto watch_ret = EtcdHelper::WatchUntilDeleted( - master_view_key.c_str(), master_view_key.size()); + mv_helper.GetMasterViewKey().c_str(), mv_helper.GetMasterViewKey().size()); if (watch_ret != ErrorCode::OK) { LOG(ERROR) << "Error watching for leadership change: " << watch_ret; diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 937d275768..8eefa92f1e 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -203,6 +203,12 @@ MasterService::MasterService(const MasterServiceConfig& config) auto etcd_oplog_store = std::make_shared(cluster_id_, /*enable_latest_seq_batch_update=*/true); oplog_manager_.SetEtcdOpLogStore(etcd_oplog_store); + // Fence against restart/promotion regressions: initialize OpLogManager + // to the maximum existing sequence_id in etcd so we don't collide/overwrite. + uint64_t max_seq = 0; + if (etcd_oplog_store->GetMaxSequenceId(max_seq) == ErrorCode::OK) { + oplog_manager_.SetInitialSequenceId(max_seq); + } LOG(INFO) << "EtcdOpLogStore initialized for cluster_id=" << cluster_id_ << " (etcd connection should be established " << "before MasterService construction)"; @@ -268,7 +274,19 @@ void MasterService::RestoreFromStandbySnapshot( const std::vector>& snapshot, uint64_t initial_oplog_sequence_id) { // 1) Ensure OpLog sequence continues without regression after failover. - oplog_manager_.SetInitialSequenceId(initial_oplog_sequence_id); + // Prefer reading the true max seq from etcd (stronger than standby_last_seq), + // fall back to caller-provided initial_oplog_sequence_id. + uint64_t start_seq = initial_oplog_sequence_id; +#ifdef STORE_USE_ETCD + if (enable_ha_ && !cluster_id_.empty()) { + EtcdOpLogStore store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + uint64_t max_seq = 0; + if (store.GetMaxSequenceId(max_seq) == ErrorCode::OK) { + start_seq = std::max(start_seq, max_seq); + } + } +#endif + oplog_manager_.SetInitialSequenceId(start_seq); // 2) Restore metadata entries. // Keep dummy allocators alive for restored memory replicas. AllocatedBuffer diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index e6851954c7..bdce1d9226 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -183,7 +183,7 @@ size_t OpLogApplier::ProcessPendingEntries() { const auto waited = std::chrono::duration_cast(now - it->second); - // Skip after 3s to avoid global stall (user requested behavior). + // Skip after timeout to avoid global stall (user requested behavior). if (waited.count() >= kMissingEntrySkipSeconds) { skipped_sequence_ids_[missing_seq] = now; missing_sequence_ids_.erase(missing_seq); @@ -192,9 +192,10 @@ size_t OpLogApplier::ProcessPendingEntries() { continue; // may skip multiple consecutive gaps } - // Optionally request from etcd after a longer wait (best-effort). - if (waited.count() >= kMissingEntryWaitSeconds) { + // Best-effort request from etcd (before skip triggers). + if (waited.count() >= kMissingEntryRequestSeconds) { missing_seq_to_request = missing_seq; + break; } break; } @@ -496,7 +497,7 @@ void OpLogApplier::ScheduleWaitForMissingEntries(uint64_t missing_seq_id) { // The actual waiting and requesting is handled in ProcessPendingEntries(). // We just log it here for tracking. VLOG(1) << "OpLogApplier: scheduling wait for missing sequence_id=" << missing_seq_id - << ", will request after " << kMissingEntryWaitSeconds << " seconds"; + << ", will request after " << kMissingEntryRequestSeconds << " seconds"; } } // namespace mooncake diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp index 8e01194958..565f611fff 100644 --- a/mooncake-store/src/oplog_watcher.cpp +++ b/mooncake-store/src/oplog_watcher.cpp @@ -47,14 +47,14 @@ bool OpLogWatcher::StartFromSequenceId(uint64_t start_seq_id) { } #ifdef STORE_USE_ETCD - uint64_t cursor_seq = start_seq_id; + uint64_t read_seq_id = start_seq_id; EtcdRevisionId last_read_rev = 0; size_t total_applied = 0; for (;;) { std::vector batch; EtcdRevisionId rev = 0; - if (!ReadOpLogSince(cursor_seq, batch, rev)) { + if (!ReadOpLogSince(read_seq_id, batch, rev)) { last_read_rev = 0; break; } @@ -63,7 +63,7 @@ bool OpLogWatcher::StartFromSequenceId(uint64_t start_seq_id) { for (const auto& e : batch) { if (applier_->ApplyOpLogEntry(e)) { last_processed_sequence_id_.store(e.sequence_id); - cursor_seq = e.sequence_id; + read_seq_id = e.sequence_id; total_applied++; } } From 7e328fa489e4a7b821a33c6d42e25b899502ac59 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 5 Jan 2026 20:31:53 +0800 Subject: [PATCH 45/59] fix gap and limit of pendingMutation and OpLogEntry checksum verify --- mooncake-store/include/master_service.h | 1 + mooncake-store/include/oplog_manager.h | 4 ++++ mooncake-store/src/master_service.cpp | 8 ++++++++ mooncake-store/src/oplog_applier.cpp | 26 ++++++++++++++++++++++--- mooncake-store/src/oplog_manager.cpp | 5 +++++ mooncake-store/src/oplog_watcher.cpp | 10 ++++++++++ 6 files changed, 51 insertions(+), 3 deletions(-) diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 7b42a3c3e1..574d050f48 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -781,6 +781,7 @@ class MasterService { std::deque pending_mutations_; std::atomic pending_mutations_running_{false}; std::thread pending_mutations_thread_; + static constexpr size_t kMaxPendingMutations = 10000; // Max queue size to prevent unbounded growth // Discarded replicas management const std::chrono::seconds put_start_discard_timeout_sec_; diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index 003b12340d..4b98e1e355 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -103,6 +103,10 @@ class OpLogManager { static uint64_t NowMs(); static uint32_t ComputeChecksum(const std::string& data); static uint32_t ComputePrefixHash(const std::string& key); + + // Verify checksum of an OpLogEntry payload. + // Returns true if checksum matches, false otherwise. + static bool VerifyChecksum(const OpLogEntry& entry); mutable std::shared_mutex mutex_; std::deque buffer_; diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 8eefa92f1e..79cd725875 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -381,6 +381,14 @@ void MasterService::EnqueuePendingMutation(PendingMutation m) { m.next_retry_at = std::chrono::steady_clock::now(); { std::lock_guard lg(pending_mutations_mutex_); + if (pending_mutations_.size() >= kMaxPendingMutations) { + // Queue full: drop oldest mutation to prevent unbounded growth. + // Log warning for monitoring. + LOG(WARNING) << "PendingMutation queue full (size=" << pending_mutations_.size() + << "), dropping oldest mutation. key=" << pending_mutations_.front().key + << ", seq=" << pending_mutations_.front().oplog_entry.sequence_id; + pending_mutations_.pop_front(); + } pending_mutations_.push_back(std::move(m)); } pending_mutations_cv_.notify_one(); diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index bdce1d9226..f9d5c4e49a 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -8,6 +8,7 @@ #include "etcd_oplog_store.h" #include "metadata_store.h" +#include "oplog_manager.h" namespace mooncake { @@ -335,10 +336,14 @@ OpLogApplier::GapResolveResult OpLogApplier::TryResolveGapsOnceForPromotion( gap_ids.erase(std::unique(gap_ids.begin(), gap_ids.end()), gap_ids.end()); r.attempted = gap_ids.size(); + std::vector successfully_processed; for (uint64_t seq : gap_ids) { OpLogEntry e; ErrorCode err = store->ReadOpLog(seq, e); if (err != ErrorCode::OK) { + // Log failed gap for monitoring, but don't clear it so it can be retried later. + LOG(WARNING) << "Promotion gap resolve: failed to fetch seq=" << seq + << ", err=" << static_cast(err); continue; } r.fetched++; @@ -347,16 +352,23 @@ OpLogApplier::GapResolveResult OpLogApplier::TryResolveGapsOnceForPromotion( if (e.op_type == OpType::REMOVE) { ApplyRemove(e); r.applied_deletes++; + successfully_processed.push_back(seq); } else if (e.op_type == OpType::PUT_REVOKE) { ApplyPutRevoke(e); r.applied_deletes++; + successfully_processed.push_back(seq); + } else { + // PUT_END or others: mark as processed (dropped) so we don't retry. + successfully_processed.push_back(seq); } } - // Clear gaps we attempted so promotion won't keep retrying them. - { + // Only clear gaps we successfully fetched and processed. + // Failed gaps remain in missing_sequence_ids_/skipped_sequence_ids_ for potential + // retry or monitoring. + if (!successfully_processed.empty()) { std::lock_guard lock(pending_mutex_); - for (uint64_t seq : gap_ids) { + for (uint64_t seq : successfully_processed) { missing_sequence_ids_.erase(seq); skipped_sequence_ids_.erase(seq); } @@ -472,6 +484,14 @@ bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) { return false; } + // Verify checksum before adding to pending entries. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLogApplier: checksum mismatch for retrieved missing entry, sequence_id=" + << missing_seq_id << ", key=" << entry.object_key + << ". Possible data corruption. Discarding entry."; + return false; + } + // Successfully retrieved the missing OpLog entry LOG(INFO) << "OpLogApplier: retrieved missing OpLog entry, sequence_id=" << missing_seq_id << ", op_type=" << static_cast(entry.op_type) diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index e4ffd85b4c..8c6dea69c3 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -142,6 +142,11 @@ uint32_t OpLogManager::ComputePrefixHash(const std::string& key) { return static_cast(XXH32(key.data(), key.size(), 0)); } +bool OpLogManager::VerifyChecksum(const OpLogEntry& entry) { + uint32_t computed = ComputeChecksum(entry.payload); + return computed == entry.checksum; +} + } // namespace mooncake diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp index 565f611fff..c354052a62 100644 --- a/mooncake-store/src/oplog_watcher.cpp +++ b/mooncake-store/src/oplog_watcher.cpp @@ -10,6 +10,7 @@ #include "etcd_helper.h" #include "etcd_oplog_store.h" #include "oplog_applier.h" +#include "oplog_manager.h" #if __has_include() #include // Ubuntu @@ -341,6 +342,15 @@ void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& v return; } + // Verify checksum to detect data corruption or tampering. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLog entry checksum mismatch: sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key + << ". Possible data corruption or tampering. Discarding entry."; + consecutive_errors_.fetch_add(1); + return; + } + // Apply the OpLog entry if (applier_->ApplyOpLogEntry(entry)) { // last_processed_sequence_id_ must be monotonic. We may "consume" duplicate From 50f1b18a00fd69d59a0c62c34d36345f3466810c Mon Sep 17 00:00:00 2001 From: BernardLee Date: Mon, 5 Jan 2026 20:34:11 +0800 Subject: [PATCH 46/59] fix --- mooncake-store/include/oplog_manager.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index 4b98e1e355..011c08fa98 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -98,15 +98,15 @@ class OpLogManager { // Current number of entries in the buffer. size_t GetEntryCount() const; + // Verify checksum of an OpLogEntry payload. + // Returns true if checksum matches, false otherwise. + // This is public so OpLogWatcher and OpLogApplier can validate entries. + static bool VerifyChecksum(const OpLogEntry& entry); private: static uint64_t NowMs(); static uint32_t ComputeChecksum(const std::string& data); static uint32_t ComputePrefixHash(const std::string& key); - - // Verify checksum of an OpLogEntry payload. - // Returns true if checksum matches, false otherwise. - static bool VerifyChecksum(const OpLogEntry& entry); mutable std::shared_mutex mutex_; std::deque buffer_; From 338a0b5e9c42e88a0b0d90b10455d00aaa0c0f81 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 09:16:30 +0800 Subject: [PATCH 47/59] fix wrap-around --- mooncake-store/include/types.h | 36 +++++++++++++++++++++++++ mooncake-store/src/etcd_oplog_store.cpp | 2 +- mooncake-store/src/oplog_applier.cpp | 12 +++++---- mooncake-store/src/oplog_watcher.cpp | 2 +- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/mooncake-store/include/types.h b/mooncake-store/include/types.h index 8fbdb806bc..b4a385e3d0 100644 --- a/mooncake-store/include/types.h +++ b/mooncake-store/include/types.h @@ -23,6 +23,42 @@ namespace mooncake { static constexpr uint64_t WRONG_VERSION = 0; static constexpr uint64_t DEFAULT_VALUE = UINT64_MAX; static constexpr uint64_t ERRNO_BASE = DEFAULT_VALUE - 1000; + +// Sequence ID comparison utilities for wrap-around safety. +// These functions use signed difference to correctly handle uint64_t overflow +// (from UINT64_MAX wrapping to 0). Assumes sequence IDs won't differ by more +// than 2^63, which is reasonable for practical systems. +// +// Example: If sequence_id wraps from UINT64_MAX to 0, then: +// IsSequenceNewer(0, UINT64_MAX) = true (0 is newer after wrap) +// IsSequenceNewer(UINT64_MAX, 0) = false (UINT64_MAX is older before wrap) +// +// Note: Using 'inline' (not 'static inline') for namespace-scope functions. +// 'inline' allows multiple definitions across translation units (ODR), +// and the linker ensures only one copy is used. 'static inline' would create +// a separate copy per translation unit, causing code bloat. +static inline bool IsSequenceNewer(uint64_t a, uint64_t b) { + // Cast to int64_t to get signed difference, then check if positive. + // This correctly handles wrap-around: if a wrapped from UINT64_MAX to 0, + // then (int64_t)(a - b) will be positive (assuming gap < 2^63). + return static_cast(a - b) > 0; +} + +static inline bool IsSequenceOlder(uint64_t a, uint64_t b) { + return static_cast(a - b) < 0; +} + +static inline bool IsSequenceEqual(uint64_t a, uint64_t b) { + return a == b; +} + +static inline bool IsSequenceNewerOrEqual(uint64_t a, uint64_t b) { + return a == b || static_cast(a - b) > 0; +} + +static inline bool IsSequenceOlderOrEqual(uint64_t a, uint64_t b) { + return a == b || static_cast(a - b) < 0; +} static constexpr uint64_t DEFAULT_DEFAULT_KV_LEASE_TTL = 5000; // in milliseconds static constexpr uint64_t DEFAULT_KV_SOFT_PIN_TTL_MS = diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp index 0f5c8222c1..94701c0566 100644 --- a/mooncake-store/src/etcd_oplog_store.cpp +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -198,7 +198,7 @@ ErrorCode EtcdOpLogStore::ReadOpLogSinceWithRevision(uint64_t start_sequence_id, } catch (...) { continue; } - if (seq <= start_sequence_id) { + if (IsSequenceOlderOrEqual(seq, start_sequence_id)) { continue; } diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index f9d5c4e49a..0ee3aefee0 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -48,7 +48,7 @@ bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { // - Those must be treated as no-op, not as "out-of-order pending", otherwise // pending_entries_ can grow and the applier may appear stuck. const uint64_t expected = expected_sequence_id_.load(); - if (entry.sequence_id < expected) { + if (IsSequenceOlder(entry.sequence_id, expected)) { // Late arrival of a previously-skipped gap entry: apply only if it's a delete/revoke. bool was_skipped = false; { @@ -82,7 +82,7 @@ bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { << ", key=" << entry.object_key; return true; // consumed (no-op) } - if (entry.sequence_id > expected) { + if (IsSequenceNewer(entry.sequence_id, expected)) { // Future entry - store into pending, wait for the gap to be filled. std::lock_guard lock(pending_mutex_); @@ -169,7 +169,7 @@ size_t OpLogApplier::ProcessPendingEntries() { } const uint64_t first_pending_seq = pending_entries_.begin()->first; const uint64_t expected = expected_sequence_id_.load(); - if (first_pending_seq <= expected) { + if (IsSequenceOlderOrEqual(first_pending_seq, expected)) { break; } @@ -225,7 +225,7 @@ size_t OpLogApplier::ProcessPendingEntries() { auto it = pending_entries_.begin(); const uint64_t expected = expected_sequence_id_.load(); - if (it->first != expected) { + if (!IsSequenceEqual(it->first, expected)) { break; // still waiting for earlier sequence_id } @@ -382,7 +382,9 @@ OpLogApplier::GapResolveResult OpLogApplier::TryResolveGapsOnceForPromotion( bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { // Only check global sequence order. - return entry.sequence_id == expected_sequence_id_.load(); + // Use IsSequenceEqual for wrap-around safety (though equality check doesn't + // need special handling, we use it for consistency). + return IsSequenceEqual(entry.sequence_id, expected_sequence_id_.load()); } void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) { diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp index c354052a62..8a63ffbe36 100644 --- a/mooncake-store/src/oplog_watcher.cpp +++ b/mooncake-store/src/oplog_watcher.cpp @@ -357,7 +357,7 @@ void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& v // / already-applied entries (entry.sequence_id < expected) as no-ops, so // never regress this counter. uint64_t cur = last_processed_sequence_id_.load(); - while (entry.sequence_id > cur && + while (IsSequenceNewer(entry.sequence_id, cur) && !last_processed_sequence_id_.compare_exchange_weak(cur, entry.sequence_id)) { // retry } From 3164aea9006795ac80bc6e345a1fe8aaca81171f Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 09:45:16 +0800 Subject: [PATCH 48/59] fix security --- mooncake-store/include/oplog_manager.h | 10 ++++++++ mooncake-store/include/types.h | 31 +++++++++++++++++++++---- mooncake-store/src/etcd_oplog_store.cpp | 14 +++++++++++ mooncake-store/src/ha_helper.cpp | 8 +++++++ mooncake-store/src/oplog_applier.cpp | 28 ++++++++++++++++++++++ mooncake-store/src/oplog_manager.cpp | 19 +++++++++++++++ mooncake-store/src/oplog_watcher.cpp | 13 +++++++++++ 7 files changed, 119 insertions(+), 4 deletions(-) diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h index 011c08fa98..df76ce6eeb 100644 --- a/mooncake-store/include/oplog_manager.h +++ b/mooncake-store/include/oplog_manager.h @@ -103,6 +103,16 @@ class OpLogManager { // This is public so OpLogWatcher and OpLogApplier can validate entries. static bool VerifyChecksum(const OpLogEntry& entry); + // Basic DoS protection for externally sourced OpLog entries (etcd watch / reads). + // Enforce conservative bounds on key/payload sizes before parsing/applying. + static constexpr size_t kMaxObjectKeySize = 4096; // 4 KiB + static constexpr size_t kMaxPayloadSize = 10 * 1024 * 1024; // 10 MiB + + // Validate OpLogEntry key/payload sizes. If invalid, returns false and + // optionally sets a human-readable reason. + static bool ValidateEntrySize(const OpLogEntry& entry, + std::string* reason = nullptr); + private: static uint64_t NowMs(); static uint32_t ComputeChecksum(const std::string& data); diff --git a/mooncake-store/include/types.h b/mooncake-store/include/types.h index b4a385e3d0..0a21a3d80d 100644 --- a/mooncake-store/include/types.h +++ b/mooncake-store/include/types.h @@ -33,10 +33,8 @@ static constexpr uint64_t ERRNO_BASE = DEFAULT_VALUE - 1000; // IsSequenceNewer(0, UINT64_MAX) = true (0 is newer after wrap) // IsSequenceNewer(UINT64_MAX, 0) = false (UINT64_MAX is older before wrap) // -// Note: Using 'inline' (not 'static inline') for namespace-scope functions. -// 'inline' allows multiple definitions across translation units (ODR), -// and the linker ensures only one copy is used. 'static inline' would create -// a separate copy per translation unit, causing code bloat. +// Note: We use 'static inline' here to give these small helpers internal +// linkage and avoid any potential ODR/linkage issues in large codebases. static inline bool IsSequenceNewer(uint64_t a, uint64_t b) { // Cast to int64_t to get signed difference, then check if positive. // This correctly handles wrap-around: if a wrapped from UINT64_MAX to 0, @@ -59,6 +57,31 @@ static inline bool IsSequenceNewerOrEqual(uint64_t a, uint64_t b) { static inline bool IsSequenceOlderOrEqual(uint64_t a, uint64_t b) { return a == b || static_cast(a - b) < 0; } + +// Cluster ID validation utilities. +// +// cluster_id is used to construct etcd key prefixes (e.g. "/oplog//..."). +// To avoid key-prefix injection / accidental cross-cluster overlap, we restrict the +// allowed characters to a conservative safe set. We validate the "component" form +// (without trailing slash). Trailing slashes should be normalized away before +// validation. +static inline bool IsValidClusterIdComponent(const std::string& cluster_id) { + if (cluster_id.empty()) { + return false; + } + if (cluster_id.size() > 128) { + return false; + } + for (unsigned char c : cluster_id) { + const bool ok = + (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || c == '_' || c == '-' || c == '.'; + if (!ok) { + return false; + } + } + return true; +} static constexpr uint64_t DEFAULT_DEFAULT_KV_LEASE_TTL = 5000; // in milliseconds static constexpr uint64_t DEFAULT_KV_SOFT_PIN_TTL_MS = diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp index 94701c0566..6d5530a172 100644 --- a/mooncake-store/src/etcd_oplog_store.cpp +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -24,6 +24,12 @@ EtcdOpLogStore::EtcdOpLogStore(const std::string& cluster_id, while (!cluster_id_.empty() && cluster_id_.back() == '/') { cluster_id_.pop_back(); } + + if (!cluster_id_.empty() && !IsValidClusterIdComponent(cluster_id_)) { + LOG(FATAL) << "Invalid cluster_id for EtcdOpLogStore: '" << cluster_id_ + << "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } + // Start batch update thread only for writers. if (enable_latest_seq_batch_update_) { batch_update_running_.store(true); @@ -430,6 +436,14 @@ bool EtcdOpLogStore::DeserializeOpLogEntry(const std::string& json_str, return false; } + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "EtcdOpLogStore: entry size rejected, sequence_id=" + << entry.sequence_id << ", key=" << entry.object_key + << ", reason=" << size_reason; + return false; + } + return true; } diff --git a/mooncake-store/src/ha_helper.cpp b/mooncake-store/src/ha_helper.cpp index 24f1e35271..b83ec2911e 100644 --- a/mooncake-store/src/ha_helper.cpp +++ b/mooncake-store/src/ha_helper.cpp @@ -35,6 +35,14 @@ void MasterViewHelper::SetClusterId(const std::string& cluster_id) { void MasterViewHelper::BuildMasterViewKeyFromClusterId( const std::string& cluster_id_in) { std::string cluster_id = cluster_id_in; + // Normalize cluster_id for validation: strip trailing slashes. + while (!cluster_id.empty() && cluster_id.back() == '/') { + cluster_id.pop_back(); + } + if (!IsValidClusterIdComponent(cluster_id)) { + LOG(FATAL) << "Invalid cluster_id for MasterViewHelper: '" << cluster_id + << "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } // Ensure the cluster_id ends with '/' if not empty if (!cluster_id.empty() && cluster_id.back() != '/') { cluster_id += '/'; diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index 0ee3aefee0..d4c0aacf09 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -41,6 +41,22 @@ EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const { } bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { + // Basic DoS protection: validate key/payload sizes before parsing/applying. + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLogApplier: entry size rejected, sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key << ", reason=" << size_reason; + return false; + } + + // Verify checksum to detect data corruption or tampering. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLogApplier: checksum mismatch, sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key + << ". Possible data corruption or tampering. Discarding entry."; + return false; + } + // Global ordering only. // // IMPORTANT: @@ -412,8 +428,12 @@ void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) { struct_json::from_json(payload, entry.payload); parse_success = true; } catch (const std::exception& e) { + const std::string prefix = + entry.payload.size() > 256 ? entry.payload.substr(0, 256) : entry.payload; LOG(ERROR) << "OpLogApplier: failed to parse payload for key=" << entry.object_key << ", sequence_id=" << entry.sequence_id + << ", payload_size=" << entry.payload.size() + << ", payload_prefix(256)=" << prefix << ", error=" << e.what(); } @@ -486,6 +506,14 @@ bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) { return false; } + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLogApplier: missing entry size rejected, sequence_id=" + << missing_seq_id << ", key=" << entry.object_key + << ", reason=" << size_reason; + return false; + } + // Verify checksum before adding to pending entries. if (!OpLogManager::VerifyChecksum(entry)) { LOG(ERROR) << "OpLogApplier: checksum mismatch for retrieved missing entry, sequence_id=" diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp index 8c6dea69c3..17d4da84ba 100644 --- a/mooncake-store/src/oplog_manager.cpp +++ b/mooncake-store/src/oplog_manager.cpp @@ -147,6 +147,25 @@ bool OpLogManager::VerifyChecksum(const OpLogEntry& entry) { return computed == entry.checksum; } +bool OpLogManager::ValidateEntrySize(const OpLogEntry& entry, + std::string* reason) { + if (entry.object_key.size() > kMaxObjectKeySize) { + if (reason) { + *reason = "object_key too large: size=" + + std::to_string(entry.object_key.size()); + } + return false; + } + if (entry.payload.size() > kMaxPayloadSize) { + if (reason) { + *reason = + "payload too large: size=" + std::to_string(entry.payload.size()); + } + return false; + } + return true; +} + } // namespace mooncake diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp index 8a63ffbe36..63a9c6d027 100644 --- a/mooncake-store/src/oplog_watcher.cpp +++ b/mooncake-store/src/oplog_watcher.cpp @@ -30,6 +30,10 @@ OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, while (!cluster_id_.empty() && cluster_id_.back() == '/') { cluster_id_.pop_back(); } + if (!cluster_id_.empty() && !IsValidClusterIdComponent(cluster_id_)) { + LOG(FATAL) << "Invalid cluster_id for OpLogWatcher: '" << cluster_id_ + << "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } } OpLogWatcher::~OpLogWatcher() { @@ -342,6 +346,15 @@ void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& v return; } + // Basic DoS protection: validate key/payload sizes before further processing. + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLog entry size rejected: sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key << ", reason=" << size_reason; + consecutive_errors_.fetch_add(1); + return; + } + // Verify checksum to detect data corruption or tampering. if (!OpLogManager::VerifyChecksum(entry)) { LOG(ERROR) << "OpLog entry checksum mismatch: sequence_id=" << entry.sequence_id From 54dd5d4d8c597a05d306ed6d1586b19f57f2c04f Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 10:01:35 +0800 Subject: [PATCH 49/59] fix cluster_id invalid --- mooncake-store/src/ha_helper.cpp | 27 ++++++++++++++++++++++----- mooncake-store/src/oplog_applier.cpp | 12 ++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/mooncake-store/src/ha_helper.cpp b/mooncake-store/src/ha_helper.cpp index b83ec2911e..09709a5978 100644 --- a/mooncake-store/src/ha_helper.cpp +++ b/mooncake-store/src/ha_helper.cpp @@ -13,14 +13,31 @@ namespace mooncake { namespace { std::string ResolveClusterIdForMasterView(const std::string& cluster_id) { + std::string resolved; if (!cluster_id.empty()) { - return cluster_id; + resolved = cluster_id; + } else { + const char* cluster_id_env = std::getenv("MC_STORE_CLUSTER_ID"); + if (cluster_id_env != nullptr && strlen(cluster_id_env) > 0) { + resolved = std::string(cluster_id_env); + } else { + resolved = DEFAULT_CLUSTER_ID; + } + } + + // Validate resolved cluster_id (even if it's the default). + // Strip trailing slashes for validation. + std::string normalized = resolved; + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); } - const char* cluster_id_env = std::getenv("MC_STORE_CLUSTER_ID"); - if (cluster_id_env != nullptr && strlen(cluster_id_env) > 0) { - return std::string(cluster_id_env); + if (!normalized.empty() && !IsValidClusterIdComponent(normalized)) { + LOG(FATAL) << "Invalid cluster_id resolved for MasterViewHelper: '" << resolved + << "' (normalized: '" << normalized + << "'). Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; } - return DEFAULT_CLUSTER_ID; + + return resolved; } } // namespace diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index d4c0aacf09..a2edf6c42d 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -20,6 +20,18 @@ OpLogApplier::OpLogApplier(MetadataStore* metadata_store, if (metadata_store_ == nullptr) { LOG(FATAL) << "OpLogApplier: metadata_store cannot be null"; } + + // Validate cluster_id if provided (required for etcd operations). + // Normalize by stripping trailing slashes for validation. + std::string normalized = cluster_id_; + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + if (!normalized.empty() && !IsValidClusterIdComponent(normalized)) { + LOG(FATAL) << "Invalid cluster_id for OpLogApplier: '" << cluster_id_ + << "' (normalized: '" << normalized + << "'). Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } } EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const { From 681826311c7569817cb38773632ff8576141e8f7 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 14:12:59 +0800 Subject: [PATCH 50/59] add state machine --- mooncake-store/include/hot_standby_service.h | 28 +- mooncake-store/include/oplog_watcher.h | 30 ++ .../include/standby_state_machine.h | 334 ++++++++++++++++++ mooncake-store/src/CMakeLists.txt | 1 + mooncake-store/src/hot_standby_service.cpp | 102 ++++-- mooncake-store/src/oplog_watcher.cpp | 6 + mooncake-store/src/standby_state_machine.cpp | 283 +++++++++++++++ 7 files changed, 758 insertions(+), 26 deletions(-) create mode 100644 mooncake-store/include/standby_state_machine.h create mode 100644 mooncake-store/src/standby_state_machine.cpp diff --git a/mooncake-store/include/hot_standby_service.h b/mooncake-store/include/hot_standby_service.h index 7b7752d66a..29544775ed 100644 --- a/mooncake-store/include/hot_standby_service.h +++ b/mooncake-store/include/hot_standby_service.h @@ -16,6 +16,7 @@ #include "oplog_manager.h" #include "oplog_watcher.h" #include "snapshot_provider.h" +#include "standby_state_machine.h" #include "types.h" namespace mooncake { @@ -51,6 +52,8 @@ struct StandbySyncStatus { std::chrono::milliseconds lag_time{0}; bool is_syncing{false}; bool is_connected{false}; + StandbyState state{StandbyState::STOPPED}; + std::chrono::milliseconds time_in_state{0}; }; /** @@ -133,6 +136,22 @@ class HotStandbyService { // Inject a snapshot provider (from external snapshot implementation). void SetSnapshotProvider(std::unique_ptr provider); + /** + * @brief Get current state from state machine + */ + StandbyState GetState() const { return state_machine_.GetState(); } + + /** + * @brief Get state machine for monitoring/debugging + */ + const StandbyStateMachine& GetStateMachine() const { return state_machine_; } + + /** + * @brief Callback for OpLogWatcher state changes + * @param event The event to process + */ + void OnWatcherEvent(StandbyEvent event); + private: /** * @brief Main replication loop (runs in background thread) @@ -205,8 +224,13 @@ class HotStandbyService { std::shared_ptr replication_stream_; std::atomic applied_seq_id_{0}; std::atomic primary_seq_id_{0}; - std::atomic running_{false}; - std::atomic is_connected_{false}; + + // State machine for managing service lifecycle + StandbyStateMachine state_machine_; + + // Helper methods for state machine + bool IsRunning() const { return state_machine_.IsRunning(); } + bool IsConnected() const { return state_machine_.IsConnected(); } // Background threads std::thread replication_thread_; diff --git a/mooncake-store/include/oplog_watcher.h b/mooncake-store/include/oplog_watcher.h index 09d9711297..9c430e87be 100644 --- a/mooncake-store/include/oplog_watcher.h +++ b/mooncake-store/include/oplog_watcher.h @@ -2,12 +2,14 @@ #include #include +#include #include #include #include #include #include "oplog_manager.h" +#include "standby_state_machine.h" #include "types.h" namespace mooncake { @@ -15,6 +17,9 @@ namespace mooncake { // Forward declaration class OpLogApplier; +// Callback type for state events +using WatcherStateCallback = std::function; + /** * @brief Watch etcd for OpLog changes and apply them to Standby * @@ -58,7 +63,29 @@ class OpLogWatcher { */ uint64_t GetLastProcessedSequenceId() const; + /** + * @brief Set callback for state events + * @param callback Callback function to invoke on state events + */ + void SetStateCallback(WatcherStateCallback callback) { + state_callback_ = std::move(callback); + } + + /** + * @brief Check if watch is healthy + */ + bool IsWatchHealthy() const { return watch_healthy_.load(); } + private: + /** + * @brief Notify state callback + */ + void NotifyStateEvent(StandbyEvent event) { + if (state_callback_) { + state_callback_(event); + } + } + bool ReadOpLogSince(uint64_t start_seq_id, std::vector& entries, EtcdRevisionId& revision_id); @@ -117,6 +144,9 @@ class OpLogWatcher { std::atomic reconnect_count_{0}; std::atomic watch_healthy_{false}; + // State callback for notifying HotStandbyService + WatcherStateCallback state_callback_; + // Constants for error handling static constexpr int kMaxConsecutiveErrors = 10; static constexpr int kReconnectDelayMs = 1000; diff --git a/mooncake-store/include/standby_state_machine.h b/mooncake-store/include/standby_state_machine.h new file mode 100644 index 0000000000..649b462566 --- /dev/null +++ b/mooncake-store/include/standby_state_machine.h @@ -0,0 +1,334 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace mooncake { + +/** + * @brief Standby service states + * + * State transition diagram: + * + * ┌─────────┐ + * │ STOPPED │◄──────────────────────────────────────┐ + * └────┬────┘ │ + * │ Start() │ Stop()/Error + * ▼ │ + * ┌─────────────┐ │ + * │ CONNECTING │◄──────────────────────┐ │ + * └──────┬──────┘ │ │ + * │ Connected │ Reconnect │ + * ▼ │ │ + * ┌─────────────┐ Error/Gap ┌────┴─────┐ │ + * │ SYNCING │─────────────────►│RECOVERING│─────┤ + * └──────┬──────┘ └──────────┘ │ + * │ Sync complete │ + * ▼ │ + * ┌─────────────┐ Watch broken ┌────────────┐ │ + * │ WATCHING │─────────────────►│RECONNECTING│───┤ + * └──────┬──────┘ └────────────┘ │ + * │ Promote() │ + * ▼ │ + * ┌─────────────┐ │ + * │ PROMOTING │───────────────────────────────────┤ + * └──────┬──────┘ │ + * │ Success │ + * ▼ │ + * ┌─────────────┐ │ + * │ PROMOTED │───────────────────────────────────┘ + * └─────────────┘ + */ +enum class StandbyState : uint8_t { + // Initial state, service not started + STOPPED = 0, + + // Connecting to etcd cluster + CONNECTING = 1, + + // Initial sync: reading historical OpLog entries + SYNCING = 2, + + // Normal operation: watching for new OpLog entries + WATCHING = 3, + + // Recovering from error: re-syncing missed entries + RECOVERING = 4, + + // Reconnecting after watch failure + RECONNECTING = 5, + + // Promotion in progress: final catch-up before becoming Primary + PROMOTING = 6, + + // Successfully promoted to Primary + PROMOTED = 7, + + // Fatal error, cannot recover + FAILED = 8, +}; + +/** + * @brief Get human-readable state name + */ +inline const char* StandbyStateToString(StandbyState state) { + switch (state) { + case StandbyState::STOPPED: + return "STOPPED"; + case StandbyState::CONNECTING: + return "CONNECTING"; + case StandbyState::SYNCING: + return "SYNCING"; + case StandbyState::WATCHING: + return "WATCHING"; + case StandbyState::RECOVERING: + return "RECOVERING"; + case StandbyState::RECONNECTING: + return "RECONNECTING"; + case StandbyState::PROMOTING: + return "PROMOTING"; + case StandbyState::PROMOTED: + return "PROMOTED"; + case StandbyState::FAILED: + return "FAILED"; + default: + return "UNKNOWN"; + } +} + +/** + * @brief Events that trigger state transitions + */ +enum class StandbyEvent : uint8_t { + // User/system actions + START, // Start() called + STOP, // Stop() called + PROMOTE, // Promote() called + + // Connection events + CONNECTED, // Successfully connected to etcd + CONNECTION_FAILED, // Failed to connect to etcd + DISCONNECTED, // Connection lost + + // Sync events + SYNC_COMPLETE, // Initial sync completed + SYNC_FAILED, // Sync failed + + // Watch events + WATCH_HEALTHY, // Watch is healthy and receiving events + WATCH_BROKEN, // Watch connection broken + + // Recovery events + RECOVERY_SUCCESS, // Successfully recovered from error + RECOVERY_FAILED, // Recovery failed + + // Promotion events + PROMOTION_SUCCESS, // Successfully promoted + PROMOTION_FAILED, // Promotion failed + + // Error events + MAX_ERRORS_REACHED, // Too many consecutive errors + FATAL_ERROR, // Unrecoverable error +}; + +inline const char* StandbyEventToString(StandbyEvent event) { + switch (event) { + case StandbyEvent::START: + return "START"; + case StandbyEvent::STOP: + return "STOP"; + case StandbyEvent::PROMOTE: + return "PROMOTE"; + case StandbyEvent::CONNECTED: + return "CONNECTED"; + case StandbyEvent::CONNECTION_FAILED: + return "CONNECTION_FAILED"; + case StandbyEvent::DISCONNECTED: + return "DISCONNECTED"; + case StandbyEvent::SYNC_COMPLETE: + return "SYNC_COMPLETE"; + case StandbyEvent::SYNC_FAILED: + return "SYNC_FAILED"; + case StandbyEvent::WATCH_HEALTHY: + return "WATCH_HEALTHY"; + case StandbyEvent::WATCH_BROKEN: + return "WATCH_BROKEN"; + case StandbyEvent::RECOVERY_SUCCESS: + return "RECOVERY_SUCCESS"; + case StandbyEvent::RECOVERY_FAILED: + return "RECOVERY_FAILED"; + case StandbyEvent::PROMOTION_SUCCESS: + return "PROMOTION_SUCCESS"; + case StandbyEvent::PROMOTION_FAILED: + return "PROMOTION_FAILED"; + case StandbyEvent::MAX_ERRORS_REACHED: + return "MAX_ERRORS_REACHED"; + case StandbyEvent::FATAL_ERROR: + return "FATAL_ERROR"; + default: + return "UNKNOWN"; + } +} + +/** + * @brief State transition result + */ +struct StateTransitionResult { + bool allowed{false}; + StandbyState old_state{StandbyState::STOPPED}; + StandbyState new_state{StandbyState::STOPPED}; + std::string reason; +}; + +/** + * @brief Callback for state transition notifications + */ +using StateChangeCallback = + std::function; + +/** + * @brief Standby State Machine + * + * Thread-safe state machine for managing Standby service lifecycle. + * All state transitions are explicit and logged. + */ +class StandbyStateMachine { + public: + StandbyStateMachine(); + + /** + * @brief Get current state (thread-safe) + */ + StandbyState GetState() const { return current_state_.load(std::memory_order_acquire); } + + /** + * @brief Check if in a specific state + */ + bool IsInState(StandbyState state) const { return GetState() == state; } + + /** + * @brief Check if service is running (SYNCING, WATCHING, RECOVERING, RECONNECTING, + * PROMOTING) + */ + bool IsRunning() const { + StandbyState s = GetState(); + return s == StandbyState::SYNCING || s == StandbyState::WATCHING || + s == StandbyState::RECOVERING || s == StandbyState::RECONNECTING || + s == StandbyState::PROMOTING; + } + + /** + * @brief Check if connected to etcd + */ + bool IsConnected() const { + StandbyState s = GetState(); + return s == StandbyState::SYNCING || s == StandbyState::WATCHING || + s == StandbyState::RECOVERING || s == StandbyState::PROMOTING; + } + + /** + * @brief Check if watch is healthy + */ + bool IsWatchHealthy() const { return GetState() == StandbyState::WATCHING; } + + /** + * @brief Check if ready for promotion + */ + bool IsReadyForPromotion() const { return GetState() == StandbyState::WATCHING; } + + /** + * @brief Process an event and perform state transition + * @param event The event to process + * @return Result indicating if transition was allowed and new state + */ + StateTransitionResult ProcessEvent(StandbyEvent event); + + /** + * @brief Register a callback for state change notifications + */ + void RegisterCallback(StateChangeCallback callback); + + /** + * @brief State transition record for debugging + */ + struct TransitionRecord { + std::chrono::steady_clock::time_point timestamp; + StandbyState from_state; + StandbyState to_state; + StandbyEvent event; + }; + + /** + * @brief Get state transition history (for debugging) + */ + std::vector GetTransitionHistory(size_t max_records = 100) const; + + /** + * @brief Get time spent in current state + */ + std::chrono::milliseconds GetTimeInCurrentState() const; + + /** + * @brief Get consecutive error count + */ + int GetConsecutiveErrors() const { return consecutive_errors_.load(); } + + /** + * @brief Increment consecutive error count + * @return New error count + */ + int IncrementErrors(); + + /** + * @brief Reset consecutive error count + */ + void ResetErrors() { consecutive_errors_.store(0); } + + /** + * @brief Get reconnect attempt count + */ + int GetReconnectCount() const { return reconnect_count_.load(); } + + /** + * @brief Increment reconnect count + */ + void IncrementReconnectCount() { reconnect_count_.fetch_add(1); } + + /** + * @brief Reset reconnect count + */ + void ResetReconnectCount() { reconnect_count_.store(0); } + + // Constants + static constexpr int kMaxConsecutiveErrors = 10; + static constexpr int kMaxReconnectAttempts = 100; + + private: + /** + * @brief Check if a transition is valid and get new state + */ + StateTransitionResult ValidateTransition(StandbyState from, StandbyEvent event) const; + + /** + * @brief Notify all registered callbacks + */ + void NotifyCallbacks(StandbyState old_state, StandbyState new_state, StandbyEvent event); + + std::atomic current_state_{StandbyState::STOPPED}; + std::atomic consecutive_errors_{0}; + std::atomic reconnect_count_{0}; + std::chrono::steady_clock::time_point state_enter_time_; + + mutable std::mutex mutex_; + std::vector callbacks_; + std::vector transition_history_; + + static constexpr size_t kMaxHistorySize = 1000; +}; + +} // namespace mooncake + diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 8f726fd780..f7f2cf32e9 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -29,6 +29,7 @@ set(MOONCAKE_STORE_SOURCES oplog_watcher.cpp oplog_applier.cpp hot_standby_service.cpp + standby_state_machine.cpp # replication_service.cpp removed - using etcd-based OpLog sync instead ) diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index c187b92733..a50eb0c94a 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -20,6 +20,16 @@ HotStandbyService::HotStandbyService(const HotStandbyConfig& config) // OpLogApplier will be created in Start() with cluster_id // For now, create without cluster_id (will be updated in Start) oplog_applier_ = std::make_unique(metadata_store_.get()); + + // Register callback for state change logging and monitoring. + // Note: callback does not capture 'this' - it only uses static functions and LOG. + // If future enhancements need member access, ensure proper lifetime management. + state_machine_.RegisterCallback([](StandbyState old_state, StandbyState new_state, StandbyEvent event) { + LOG(INFO) << "HotStandbyService state changed: " + << StandbyStateToString(old_state) << " -> " + << StandbyStateToString(new_state) + << " (event: " << StandbyEventToString(event) << ")"; + }); } // StandbyMetadataStore implementation @@ -92,11 +102,19 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, const std::string& cluster_id) { std::lock_guard lock(mutex_); - if (running_.load()) { + // Use state machine to check if already running + if (IsRunning()) { LOG(WARNING) << "HotStandbyService is already running"; return ErrorCode::OK; } + // Trigger START event + auto result = state_machine_.ProcessEvent(StandbyEvent::START); + if (!result.allowed) { + LOG(ERROR) << "Cannot start HotStandbyService: " << result.reason; + return ErrorCode::INVALID_STATE; + } + config_.primary_address = primary_address; etcd_endpoints_ = etcd_endpoints; cluster_id_ = cluster_id; @@ -106,9 +124,13 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(etcd_endpoints.c_str()); if (err != ErrorCode::OK) { LOG(ERROR) << "Failed to connect to etcd: " << etcd_endpoints; + state_machine_.ProcessEvent(StandbyEvent::CONNECTION_FAILED); return err; } + // Transition to SYNCING state + state_machine_.ProcessEvent(StandbyEvent::CONNECTED); + // Preserve existing local state if HotStandbyService is restarted in-process: // - metadata_store_ may already contain real-time metadata // - oplog_applier_ may already have expected_sequence_id_ @@ -131,12 +153,14 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, oplog_applier_->Recover(local_last_seq_id); } - // Create OpLogWatcher + // Create OpLogWatcher with state machine callback oplog_watcher_ = std::make_unique( etcd_endpoints, cluster_id, oplog_applier_.get()); - - running_.store(true); - is_connected_.store(true); + + // Register callback for watcher events + oplog_watcher_->SetStateCallback([this](StandbyEvent event) { + OnWatcherEvent(event); + }); // Bootstrap: // - If we already have local state (warm start), do NOT reload snapshot. @@ -170,6 +194,10 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, if (!oplog_watcher_->StartFromSequenceId(last_applied_seq_id)) { LOG(WARNING) << "Failed to start OpLogWatcher from sequence_id=" << last_applied_seq_id << ", continuing anyway"; + state_machine_.ProcessEvent(StandbyEvent::SYNC_FAILED); + } else { + // Transition to WATCHING state after successful sync + state_machine_.ProcessEvent(StandbyEvent::SYNC_COMPLETE); } // Start background threads @@ -180,21 +208,26 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, } LOG(INFO) << "HotStandbyService started, watching etcd OpLog for cluster: " - << cluster_id; + << cluster_id << ", state=" << StandbyStateToString(GetState()); return ErrorCode::OK; #else + state_machine_.ProcessEvent(StandbyEvent::FATAL_ERROR); LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot start HotStandbyService"; return ErrorCode::INTERNAL_ERROR; #endif } +void HotStandbyService::OnWatcherEvent(StandbyEvent event) { + state_machine_.ProcessEvent(event); +} + void HotStandbyService::Stop() { - if (!running_.load()) { + if (!IsRunning() && GetState() != StandbyState::PROMOTING) { return; } - running_.store(false); - is_connected_.store(false); + // Trigger STOP event + state_machine_.ProcessEvent(StandbyEvent::STOP); // Stop OpLogWatcher if (oplog_watcher_) { @@ -210,7 +243,7 @@ void HotStandbyService::Stop() { verification_thread_.join(); } - LOG(INFO) << "HotStandbyService stopped"; + LOG(INFO) << "HotStandbyService stopped, final_state=" << StandbyStateToString(GetState()); } StandbySyncStatus HotStandbyService::GetSyncStatus() const { @@ -228,7 +261,11 @@ StandbySyncStatus HotStandbyService::GetSyncStatus() const { // Primary sequence ID (best-effort): updated by ReplicationLoop via etcd `/latest`. status.primary_seq_id = primary_seq_id_.load(); - status.is_connected = is_connected_.load(); + + // Use state machine for connection status + status.is_connected = IsConnected(); + status.state = GetState(); + status.time_in_state = state_machine_.GetTimeInCurrentState(); if (status.primary_seq_id > status.applied_seq_id) { status.lag_entries = status.primary_seq_id - status.applied_seq_id; @@ -239,17 +276,19 @@ StandbySyncStatus HotStandbyService::GetSyncStatus() const { // Calculate lag time (placeholder - in full implementation this would // track actual time differences) status.lag_time = std::chrono::milliseconds(0); - status.is_syncing = running_.load() && is_connected_.load(); + status.is_syncing = IsRunning() && IsConnected(); return status; } bool HotStandbyService::IsReadyForPromotion() const { - StandbySyncStatus status = GetSyncStatus(); - if (!status.is_connected) { + // Use state machine to check if ready for promotion + if (!state_machine_.IsReadyForPromotion()) { return false; } + StandbySyncStatus status = GetSyncStatus(); + // Allow promotion even with large lag - the new Primary can continue // syncing remaining OpLog entries from etcd after promotion. // Log a warning if lag is large, but don't block promotion. @@ -267,7 +306,15 @@ std::unique_ptr HotStandbyService::Promote() { std::lock_guard lock(mutex_); if (!IsReadyForPromotion()) { - LOG(ERROR) << "Standby is not ready for promotion (not connected)"; + LOG(ERROR) << "Standby is not ready for promotion, state=" + << StandbyStateToString(GetState()); + return nullptr; + } + + // Trigger PROMOTE event + auto result = state_machine_.ProcessEvent(StandbyEvent::PROMOTE); + if (!result.allowed) { + LOG(ERROR) << "Cannot promote: " << result.reason; return nullptr; } @@ -276,7 +323,8 @@ std::unique_ptr HotStandbyService::Promote() { LOG(INFO) << "Promoting Standby to Primary. Applied seq_id: " << current_applied_seq_id - << ", lag: " << status.lag_entries << " entries"; + << ", lag: " << status.lag_entries << " entries" + << ", state: " << StandbyStateToString(GetState()); // Final catch-up sync before promotion. // IMPORTANT: @@ -320,7 +368,11 @@ std::unique_ptr HotStandbyService::Promote() { } LOG(INFO) << "Final catch-up sync done. total_applied=" << total_applied; + // Transition to PROMOTED state + state_machine_.ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + // Stop replication (OpLogWatcher will stop watching) + // Note: This will trigger STOP event, transitioning to STOPPED Stop(); // In full implementation, we would: @@ -386,8 +438,8 @@ void HotStandbyService::ReplicationLoop() { // in its own thread. This loop now just monitors the status and updates // metrics. - while (running_.load()) { - if (!is_connected_.load()) { + while (IsRunning()) { + if (!IsConnected()) { // Not connected - wait a bit before checking again std::this_thread::sleep_for(std::chrono::seconds(1)); continue; @@ -424,11 +476,11 @@ void HotStandbyService::ReplicationLoop() { void HotStandbyService::VerificationLoop() { LOG(INFO) << "Verification loop started"; - while (running_.load()) { + while (IsRunning()) { std::this_thread::sleep_for( std::chrono::seconds(config_.verification_interval_sec)); - if (!is_connected_.load()) { + if (!IsConnected()) { continue; } @@ -439,7 +491,8 @@ void HotStandbyService::VerificationLoop() { // 4. Handle mismatches if any // Placeholder: Log that verification would happen - VLOG(1) << "Verification check (placeholder)"; + VLOG(1) << "Verification check (placeholder), state=" + << StandbyStateToString(GetState()); } LOG(INFO) << "Verification loop stopped"; @@ -477,10 +530,11 @@ bool HotStandbyService::ConnectToPrimary() { void HotStandbyService::DisconnectFromPrimary() { // With etcd-based OpLog sync, disconnection is handled by OpLogWatcher // This method is kept for compatibility - if (is_connected_.load()) { - is_connected_.store(false); + if (IsConnected()) { + state_machine_.ProcessEvent(StandbyEvent::DISCONNECTED); replication_stream_.reset(); - LOG(INFO) << "Disconnected from Primary (etcd-based sync)"; + LOG(INFO) << "Disconnected from Primary (etcd-based sync), state=" + << StandbyStateToString(GetState()); } } diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp index 63a9c6d027..918a086ae0 100644 --- a/mooncake-store/src/oplog_watcher.cpp +++ b/mooncake-store/src/oplog_watcher.cpp @@ -183,6 +183,7 @@ void OpLogWatcher::WatchOpLog() { LOG(ERROR) << "Failed to start watch for prefix " << watch_prefix << ", error=" << static_cast(err); watch_healthy_.store(false); + NotifyStateEvent(StandbyEvent::WATCH_BROKEN); // Try to reconnect TryReconnect(); @@ -192,6 +193,7 @@ void OpLogWatcher::WatchOpLog() { LOG(INFO) << "Watch started for prefix " << watch_prefix; watch_healthy_.store(true); consecutive_errors_.store(0); + NotifyStateEvent(StandbyEvent::WATCH_HEALTHY); // The watch is now running in the background (via Go goroutine) // We just need to keep the thread alive until Stop() is called or watch fails @@ -208,6 +210,7 @@ void OpLogWatcher::WatchOpLog() { LOG(WARNING) << "Too many consecutive errors (" << consecutive_errors_.load() << "), reconnecting watch..."; watch_healthy_.store(false); + NotifyStateEvent(StandbyEvent::MAX_ERRORS_REACHED); break; } } @@ -215,6 +218,7 @@ void OpLogWatcher::WatchOpLog() { if (running_.load() && !watch_healthy_.load()) { // Cancel current watch before reconnecting EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + NotifyStateEvent(StandbyEvent::WATCH_BROKEN); TryReconnect(); } } @@ -244,8 +248,10 @@ void OpLogWatcher::TryReconnect() { // Sync any missed entries before resuming watch if (SyncMissedEntries()) { LOG(INFO) << "Successfully synced missed OpLog entries"; + NotifyStateEvent(StandbyEvent::RECOVERY_SUCCESS); } else { LOG(WARNING) << "Failed to sync missed OpLog entries, continuing anyway"; + NotifyStateEvent(StandbyEvent::RECOVERY_FAILED); } } diff --git a/mooncake-store/src/standby_state_machine.cpp b/mooncake-store/src/standby_state_machine.cpp new file mode 100644 index 0000000000..616649a454 --- /dev/null +++ b/mooncake-store/src/standby_state_machine.cpp @@ -0,0 +1,283 @@ +#include "standby_state_machine.h" + +#include + +namespace mooncake { + +StandbyStateMachine::StandbyStateMachine() + : state_enter_time_(std::chrono::steady_clock::now()) {} + +StateTransitionResult StandbyStateMachine::ValidateTransition(StandbyState from, + StandbyEvent event) const { + StateTransitionResult result; + result.allowed = false; + result.old_state = from; + result.new_state = from; + + // State transition table + switch (from) { + case StandbyState::STOPPED: + if (event == StandbyEvent::START) { + result.allowed = true; + result.new_state = StandbyState::CONNECTING; + } + break; + + case StandbyState::CONNECTING: + switch (event) { + case StandbyEvent::CONNECTED: + result.allowed = true; + result.new_state = StandbyState::SYNCING; + break; + case StandbyEvent::CONNECTION_FAILED: + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::SYNCING: + switch (event) { + case StandbyEvent::SYNC_COMPLETE: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + case StandbyEvent::SYNC_FAILED: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + default: + break; + } + break; + + case StandbyState::WATCHING: + switch (event) { + case StandbyEvent::WATCH_BROKEN: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::MAX_ERRORS_REACHED: + result.allowed = true; + result.new_state = StandbyState::RECOVERING; + break; + case StandbyEvent::PROMOTE: + result.allowed = true; + result.new_state = StandbyState::PROMOTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + // WATCH_HEALTHY in WATCHING state is a no-op (stay in WATCHING) + case StandbyEvent::WATCH_HEALTHY: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + default: + break; + } + break; + + case StandbyState::RECOVERING: + switch (event) { + case StandbyEvent::RECOVERY_SUCCESS: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + case StandbyEvent::RECOVERY_FAILED: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + default: + break; + } + break; + + case StandbyState::RECONNECTING: + switch (event) { + case StandbyEvent::CONNECTED: + result.allowed = true; + result.new_state = StandbyState::SYNCING; + break; + case StandbyEvent::MAX_ERRORS_REACHED: + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::PROMOTING: + switch (event) { + case StandbyEvent::PROMOTION_SUCCESS: + result.allowed = true; + result.new_state = StandbyState::PROMOTED; + break; + case StandbyEvent::PROMOTION_FAILED: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::PROMOTED: + if (event == StandbyEvent::STOP) { + result.allowed = true; + result.new_state = StandbyState::STOPPED; + } + break; + + case StandbyState::FAILED: + if (event == StandbyEvent::STOP) { + result.allowed = true; + result.new_state = StandbyState::STOPPED; + } else if (event == StandbyEvent::START) { + // Allow restart from FAILED state + result.allowed = true; + result.new_state = StandbyState::CONNECTING; + } + break; + } + + if (!result.allowed) { + result.reason = std::string("Invalid transition from ") + StandbyStateToString(from) + + " on event " + StandbyEventToString(event); + } + + return result; +} + +StateTransitionResult StandbyStateMachine::ProcessEvent(StandbyEvent event) { + StandbyState old_state = current_state_.load(std::memory_order_acquire); + StateTransitionResult result = ValidateTransition(old_state, event); + + if (result.allowed && result.new_state != old_state) { + std::lock_guard lock(mutex_); + + // Double-check state hasn't changed (compare-and-swap pattern) + StandbyState current = current_state_.load(std::memory_order_acquire); + if (current != old_state) { + // State changed by another thread, re-validate + result = ValidateTransition(current, event); + old_state = current; + result.old_state = current; + if (!result.allowed || result.new_state == old_state) { + return result; + } + } + + // Record transition + TransitionRecord record; + record.timestamp = std::chrono::steady_clock::now(); + record.from_state = old_state; + record.to_state = result.new_state; + record.event = event; + + transition_history_.push_back(record); + if (transition_history_.size() > kMaxHistorySize) { + transition_history_.erase(transition_history_.begin()); + } + + // Update state + current_state_.store(result.new_state, std::memory_order_release); + state_enter_time_ = record.timestamp; + + LOG(INFO) << "Standby state transition: " << StandbyStateToString(old_state) << " -> " + << StandbyStateToString(result.new_state) + << " (event: " << StandbyEventToString(event) << ")"; + + // Make a copy of callbacks to release the lock before calling them + std::vector callbacks_copy = callbacks_; + + // Release lock by ending scope, then notify callbacks + // Note: We need to unlock before calling callbacks to avoid deadlock + // So we copy callbacks and call after the lock_guard scope ends + for (const auto& callback : callbacks_copy) { + if (callback) { + callback(old_state, result.new_state, event); + } + } + } else if (!result.allowed) { + VLOG(1) << "Standby state transition rejected: " << result.reason; + } + + return result; +} + +void StandbyStateMachine::RegisterCallback(StateChangeCallback callback) { + std::lock_guard lock(mutex_); + callbacks_.push_back(std::move(callback)); +} + +std::vector StandbyStateMachine::GetTransitionHistory( + size_t max_records) const { + std::lock_guard lock(mutex_); + + if (transition_history_.size() <= max_records) { + return transition_history_; + } + + return std::vector(transition_history_.end() - max_records, + transition_history_.end()); +} + +std::chrono::milliseconds StandbyStateMachine::GetTimeInCurrentState() const { + auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(mutex_); + return std::chrono::duration_cast(now - state_enter_time_); +} + +int StandbyStateMachine::IncrementErrors() { + int new_count = consecutive_errors_.fetch_add(1) + 1; + if (new_count >= kMaxConsecutiveErrors) { + // Trigger MAX_ERRORS_REACHED event + ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + } + return new_count; +} + +} // namespace mooncake + From c3edf2d2c662a8fce01507b779853b6e26f2c3c7 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 14:20:37 +0800 Subject: [PATCH 51/59] fix compile --- mooncake-store/src/hot_standby_service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index a50eb0c94a..8238dd02b7 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -112,7 +112,7 @@ ErrorCode HotStandbyService::Start(const std::string& primary_address, auto result = state_machine_.ProcessEvent(StandbyEvent::START); if (!result.allowed) { LOG(ERROR) << "Cannot start HotStandbyService: " << result.reason; - return ErrorCode::INVALID_STATE; + return ErrorCode::INTERNAL_ERROR; // State machine rejected START } config_.primary_address = primary_address; From 6905b5cf37728790063f126952c8481cbed0ee9a Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 15:01:12 +0800 Subject: [PATCH 52/59] fix retry --- mooncake-store/src/hot_standby_service.cpp | 63 ++++++++++++++++++---- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index 8238dd02b7..f7111224ec 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -334,28 +334,69 @@ std::unique_ptr HotStandbyService::Promote() { oplog_watcher_->Stop(); } - // Best-effort: resolve any outstanding gaps ONCE before promotion. - // Do NOT block promotion if gaps cannot be fetched. + // Best-effort: resolve any outstanding gaps with retry before promotion. + // Do NOT block promotion if gaps cannot be fetched after max retries. + static constexpr int kMaxGapResolveRetries = 3; if (oplog_applier_) { - auto res = oplog_applier_->TryResolveGapsOnceForPromotion(/*max_ids=*/1024); - if (res.attempted > 0) { - LOG(INFO) << "Promotion gap resolve (best-effort): attempted=" << res.attempted + for (int retry = 0; retry < kMaxGapResolveRetries; ++retry) { + auto res = oplog_applier_->TryResolveGapsOnceForPromotion(/*max_ids=*/1024); + if (res.attempted == 0) { + // No gaps to resolve + break; + } + LOG(INFO) << "Promotion gap resolve (attempt " << (retry + 1) << "/" + << kMaxGapResolveRetries << "): attempted=" << res.attempted << ", fetched=" << res.fetched << ", applied_deletes=" << res.applied_deletes; + if (res.fetched == res.attempted) { + // All gaps resolved successfully + break; + } + // Some gaps failed, retry after short delay + if (retry + 1 < kMaxGapResolveRetries) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } } } LOG(INFO) << "Final catch-up sync from etcd before promotion..."; EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); const size_t batch_size = 1000; - uint64_t start_seq = current_applied_seq_id + 1; + + // P0 fix: Prevent underflow when current_applied_seq_id is 0 + // ReadOpLogSince reads entries with seq > given_seq, so we pass current_applied_seq_id directly + uint64_t read_from_seq = current_applied_seq_id; // Will read entries with seq > read_from_seq + + // P1 fix: Add timeout control to prevent infinite blocking + static constexpr size_t kMaxCatchUpBatches = 100; // Max 100 batches * 1000 = 100k entries + static constexpr auto kMaxCatchUpDuration = std::chrono::seconds(30); + auto catch_up_start = std::chrono::steady_clock::now(); + size_t total_applied = 0; + size_t batch_count = 0; + for (;;) { + // Check timeout + auto elapsed = std::chrono::steady_clock::now() - catch_up_start; + if (elapsed > kMaxCatchUpDuration) { + LOG(WARNING) << "Final catch-up: timeout after " + << std::chrono::duration_cast(elapsed).count() + << "s. Proceeding with promotion. total_applied=" << total_applied; + break; + } + + // Check batch limit + if (batch_count >= kMaxCatchUpBatches) { + LOG(WARNING) << "Final catch-up: reached max batch limit (" << kMaxCatchUpBatches + << "). Proceeding with promotion. total_applied=" << total_applied; + break; + } + std::vector batch; - ErrorCode read_err = oplog_store.ReadOpLogSince(start_seq - 1, batch_size, batch); + ErrorCode read_err = oplog_store.ReadOpLogSince(read_from_seq, batch_size, batch); if (read_err != ErrorCode::OK) { LOG(WARNING) << "Final catch-up: failed to read OpLog since seq=" - << (start_seq - 1) << ", err=" << read_err + << read_from_seq << ", err=" << static_cast(read_err) << ". Proceeding with promotion."; break; } @@ -364,9 +405,11 @@ std::unique_ptr HotStandbyService::Promote() { } size_t applied = oplog_applier_->ApplyOpLogEntries(batch); total_applied += applied; - start_seq = batch.back().sequence_id + 1; + read_from_seq = batch.back().sequence_id; // Next read will get entries > this seq + ++batch_count; } - LOG(INFO) << "Final catch-up sync done. total_applied=" << total_applied; + LOG(INFO) << "Final catch-up sync done. total_applied=" << total_applied + << ", batches=" << batch_count; // Transition to PROMOTED state state_machine_.ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); From e05354a943120b40ea4f5c0e8582b320cfed6414 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 15:21:24 +0800 Subject: [PATCH 53/59] ha metrics --- mooncake-store/include/ha_metric_manager.h | 192 +++++++++++++++ mooncake-store/src/CMakeLists.txt | 1 + mooncake-store/src/ha_metric_manager.cpp | 274 +++++++++++++++++++++ mooncake-store/src/hot_standby_service.cpp | 14 +- mooncake-store/src/master_service.cpp | 23 ++ mooncake-store/src/oplog_applier.cpp | 21 ++ mooncake-store/src/oplog_watcher.cpp | 2 + mooncake-store/src/rpc_service.cpp | 18 ++ 8 files changed, 542 insertions(+), 3 deletions(-) create mode 100644 mooncake-store/include/ha_metric_manager.h create mode 100644 mooncake-store/src/ha_metric_manager.cpp diff --git a/mooncake-store/include/ha_metric_manager.h b/mooncake-store/include/ha_metric_manager.h new file mode 100644 index 0000000000..feebcc59ed --- /dev/null +++ b/mooncake-store/include/ha_metric_manager.h @@ -0,0 +1,192 @@ +#pragma once + +#include +#include +#include +#include + +#include "ylt/metric/counter.hpp" +#include "ylt/metric/gauge.hpp" +#include "ylt/metric/histogram.hpp" + +namespace mooncake { + +/** + * @brief Singleton manager for High Availability (HA) related metrics. + * + * This class provides metrics for monitoring the health and performance + * of the OpLog replication system, including: + * - OpLog sequence tracking + * - Standby replication lag + * - Error counters (checksum failures, skipped entries) + * - Performance histograms (etcd write latency) + * - Queue sizes (pending mutations) + */ +class HAMetricManager { + public: + // --- Singleton Access --- + static HAMetricManager& instance(); + + HAMetricManager(const HAMetricManager&) = delete; + HAMetricManager& operator=(const HAMetricManager&) = delete; + HAMetricManager(HAMetricManager&&) = delete; + HAMetricManager& operator=(HAMetricManager&&) = delete; + + // ========== OpLog Sequence Metrics (Gauge) ========== + + /** + * @brief Set the latest OpLog sequence ID on Primary + */ + void set_oplog_last_sequence_id(int64_t seq_id); + int64_t get_oplog_last_sequence_id(); + + /** + * @brief Set the Standby's applied sequence ID + */ + void set_oplog_applied_sequence_id(int64_t seq_id); + int64_t get_oplog_applied_sequence_id(); + + /** + * @brief Set the replication lag (entries behind Primary) + */ + void set_oplog_standby_lag(int64_t lag); + int64_t get_oplog_standby_lag(); + + /** + * @brief Set the number of pending (out-of-order) entries in OpLogApplier + */ + void set_oplog_pending_entries(int64_t count); + int64_t get_oplog_pending_entries(); + + /** + * @brief Set the pending mutation queue size (retry queue) + */ + void set_pending_mutation_queue_size(int64_t size); + int64_t get_pending_mutation_queue_size(); + + // ========== Error Counters ========== + + /** + * @brief Increment counter for skipped OpLog entries + */ + void inc_oplog_skipped_entries(int64_t val = 1); + int64_t get_oplog_skipped_entries_total(); + + /** + * @brief Increment counter for checksum verification failures + */ + void inc_oplog_checksum_failures(int64_t val = 1); + int64_t get_oplog_checksum_failures_total(); + + /** + * @brief Increment counter for gap resolve attempts + */ + void inc_oplog_gap_resolve_attempts(int64_t val = 1); + int64_t get_oplog_gap_resolve_attempts_total(); + + /** + * @brief Increment counter for successful gap resolves + */ + void inc_oplog_gap_resolve_success(int64_t val = 1); + int64_t get_oplog_gap_resolve_success_total(); + + /** + * @brief Increment counter for etcd write failures + */ + void inc_oplog_etcd_write_failures(int64_t val = 1); + int64_t get_oplog_etcd_write_failures_total(); + + /** + * @brief Increment counter for etcd write retries + */ + void inc_oplog_etcd_write_retries(int64_t val = 1); + int64_t get_oplog_etcd_write_retries_total(); + + /** + * @brief Increment counter for watch disconnections + */ + void inc_oplog_watch_disconnections(int64_t val = 1); + int64_t get_oplog_watch_disconnections_total(); + + /** + * @brief Increment counter for successfully applied OpLog entries + */ + void inc_oplog_applied_entries(int64_t val = 1); + int64_t get_oplog_applied_entries_total(); + + // ========== Latency Histograms ========== + + /** + * @brief Record etcd write latency in microseconds + */ + void observe_oplog_etcd_write_latency_us(int64_t latency_us); + + /** + * @brief Record OpLog apply latency in microseconds + */ + void observe_oplog_apply_latency_us(int64_t latency_us); + + // ========== State Machine Metrics ========== + + /** + * @brief Set the current Standby state (as integer for Prometheus) + * @param state_value Integer representation of StandbyState + */ + void set_standby_state(int64_t state_value); + int64_t get_standby_state(); + + /** + * @brief Increment state transition counter + */ + void inc_state_transitions(int64_t val = 1); + int64_t get_state_transitions_total(); + + // ========== Serialization ========== + + /** + * @brief Serializes all HA metrics into Prometheus text format. + * @return A string containing the metrics in Prometheus format. + */ + std::string serialize_metrics(); + + /** + * @brief Generates a concise, human-readable summary of HA metrics. + * @return A string containing the formatted summary. + */ + std::string get_summary_string(); + + private: + // --- Private Constructor & Destructor --- + HAMetricManager(); + ~HAMetricManager() = default; + + // --- Metric Members --- + + // OpLog Sequence Gauges + ylt::metric::gauge_t oplog_last_sequence_id_; + ylt::metric::gauge_t oplog_applied_sequence_id_; + ylt::metric::gauge_t oplog_standby_lag_; + ylt::metric::gauge_t oplog_pending_entries_; + ylt::metric::gauge_t pending_mutation_queue_size_; + + // Error Counters + ylt::metric::counter_t oplog_skipped_entries_total_; + ylt::metric::counter_t oplog_checksum_failures_total_; + ylt::metric::counter_t oplog_gap_resolve_attempts_total_; + ylt::metric::counter_t oplog_gap_resolve_success_total_; + ylt::metric::counter_t oplog_etcd_write_failures_total_; + ylt::metric::counter_t oplog_etcd_write_retries_total_; + ylt::metric::counter_t oplog_watch_disconnections_total_; + ylt::metric::counter_t oplog_applied_entries_total_; + + // Latency Histograms (buckets in microseconds: 100us, 500us, 1ms, 5ms, 10ms, 50ms, 100ms, 500ms, 1s) + ylt::metric::histogram_t oplog_etcd_write_latency_us_; + ylt::metric::histogram_t oplog_apply_latency_us_; + + // State Machine + ylt::metric::gauge_t standby_state_; + ylt::metric::counter_t state_transitions_total_; +}; + +} // namespace mooncake + diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index f7f2cf32e9..7d4976e891 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -30,6 +30,7 @@ set(MOONCAKE_STORE_SOURCES oplog_applier.cpp hot_standby_service.cpp standby_state_machine.cpp + ha_metric_manager.cpp # replication_service.cpp removed - using etcd-based OpLog sync instead ) diff --git a/mooncake-store/src/ha_metric_manager.cpp b/mooncake-store/src/ha_metric_manager.cpp new file mode 100644 index 0000000000..645206715d --- /dev/null +++ b/mooncake-store/src/ha_metric_manager.cpp @@ -0,0 +1,274 @@ +#include "ha_metric_manager.h" + +#include + +#include +#include + +namespace mooncake { + +// --- Singleton Instance --- +HAMetricManager& HAMetricManager::instance() { + static HAMetricManager static_instance; + return static_instance; +} + +// --- Constructor --- +HAMetricManager::HAMetricManager() + // OpLog Sequence Gauges + : oplog_last_sequence_id_( + "ha_oplog_last_sequence_id", + "Latest OpLog sequence ID written by Primary"), + oplog_applied_sequence_id_( + "ha_oplog_applied_sequence_id", + "Latest OpLog sequence ID applied by Standby"), + oplog_standby_lag_( + "ha_oplog_standby_lag", + "Number of OpLog entries Standby is behind Primary"), + oplog_pending_entries_( + "ha_oplog_pending_entries", + "Number of out-of-order entries waiting in OpLogApplier"), + pending_mutation_queue_size_( + "ha_pending_mutation_queue_size", + "Number of mutations pending etcd write retry"), + + // Error Counters + oplog_skipped_entries_total_( + "ha_oplog_skipped_entries_total", + "Total number of OpLog entries skipped due to timeout"), + oplog_checksum_failures_total_( + "ha_oplog_checksum_failures_total", + "Total number of OpLog entries with checksum verification failures"), + oplog_gap_resolve_attempts_total_( + "ha_oplog_gap_resolve_attempts_total", + "Total number of attempts to resolve missing OpLog entries"), + oplog_gap_resolve_success_total_( + "ha_oplog_gap_resolve_success_total", + "Total number of successfully resolved missing OpLog entries"), + oplog_etcd_write_failures_total_( + "ha_oplog_etcd_write_failures_total", + "Total number of failed etcd write operations"), + oplog_etcd_write_retries_total_( + "ha_oplog_etcd_write_retries_total", + "Total number of etcd write retry attempts"), + oplog_watch_disconnections_total_( + "ha_oplog_watch_disconnections_total", + "Total number of OpLog watch disconnections"), + oplog_applied_entries_total_( + "ha_oplog_applied_entries_total", + "Total number of OpLog entries successfully applied"), + + // Latency Histograms (buckets in microseconds) + // 100us, 500us, 1ms, 5ms, 10ms, 50ms, 100ms, 500ms, 1s, 5s + oplog_etcd_write_latency_us_( + "ha_oplog_etcd_write_latency_us", + "Latency of etcd write operations in microseconds", + {100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000, 5000000}), + oplog_apply_latency_us_( + "ha_oplog_apply_latency_us", + "Latency of OpLog entry application in microseconds", + {10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000}), + + // State Machine + standby_state_( + "ha_standby_state", + "Current state of the Standby service (0=STOPPED, 1=CONNECTING, " + "2=SYNCING, 3=WATCHING, 4=RECOVERING, 5=RECONNECTING, " + "6=PROMOTING, 7=PROMOTED, 8=FAILED)"), + state_transitions_total_( + "ha_state_transitions_total", + "Total number of Standby state machine transitions") { + // Initialize gauges to 0 for proper Prometheus output + oplog_last_sequence_id_.update(0); + oplog_applied_sequence_id_.update(0); + oplog_standby_lag_.update(0); + oplog_pending_entries_.update(0); + pending_mutation_queue_size_.update(0); + standby_state_.update(0); +} + +// ========== OpLog Sequence Metrics (Gauge) ========== + +void HAMetricManager::set_oplog_last_sequence_id(int64_t seq_id) { + oplog_last_sequence_id_.update(seq_id); +} + +int64_t HAMetricManager::get_oplog_last_sequence_id() { + return static_cast(oplog_last_sequence_id_.value()); +} + +void HAMetricManager::set_oplog_applied_sequence_id(int64_t seq_id) { + oplog_applied_sequence_id_.update(seq_id); +} + +int64_t HAMetricManager::get_oplog_applied_sequence_id() { + return static_cast(oplog_applied_sequence_id_.value()); +} + +void HAMetricManager::set_oplog_standby_lag(int64_t lag) { + oplog_standby_lag_.update(lag); +} + +int64_t HAMetricManager::get_oplog_standby_lag() { + return static_cast(oplog_standby_lag_.value()); +} + +void HAMetricManager::set_oplog_pending_entries(int64_t count) { + oplog_pending_entries_.update(count); +} + +int64_t HAMetricManager::get_oplog_pending_entries() { + return static_cast(oplog_pending_entries_.value()); +} + +void HAMetricManager::set_pending_mutation_queue_size(int64_t size) { + pending_mutation_queue_size_.update(size); +} + +int64_t HAMetricManager::get_pending_mutation_queue_size() { + return static_cast(pending_mutation_queue_size_.value()); +} + +// ========== Error Counters ========== + +void HAMetricManager::inc_oplog_skipped_entries(int64_t val) { + oplog_skipped_entries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_skipped_entries_total() { + return static_cast(oplog_skipped_entries_total_.value()); +} + +void HAMetricManager::inc_oplog_checksum_failures(int64_t val) { + oplog_checksum_failures_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_checksum_failures_total() { + return static_cast(oplog_checksum_failures_total_.value()); +} + +void HAMetricManager::inc_oplog_gap_resolve_attempts(int64_t val) { + oplog_gap_resolve_attempts_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_gap_resolve_attempts_total() { + return static_cast(oplog_gap_resolve_attempts_total_.value()); +} + +void HAMetricManager::inc_oplog_gap_resolve_success(int64_t val) { + oplog_gap_resolve_success_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_gap_resolve_success_total() { + return static_cast(oplog_gap_resolve_success_total_.value()); +} + +void HAMetricManager::inc_oplog_etcd_write_failures(int64_t val) { + oplog_etcd_write_failures_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_etcd_write_failures_total() { + return static_cast(oplog_etcd_write_failures_total_.value()); +} + +void HAMetricManager::inc_oplog_etcd_write_retries(int64_t val) { + oplog_etcd_write_retries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_etcd_write_retries_total() { + return static_cast(oplog_etcd_write_retries_total_.value()); +} + +void HAMetricManager::inc_oplog_watch_disconnections(int64_t val) { + oplog_watch_disconnections_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_watch_disconnections_total() { + return static_cast(oplog_watch_disconnections_total_.value()); +} + +void HAMetricManager::inc_oplog_applied_entries(int64_t val) { + oplog_applied_entries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_applied_entries_total() { + return static_cast(oplog_applied_entries_total_.value()); +} + +// ========== Latency Histograms ========== + +void HAMetricManager::observe_oplog_etcd_write_latency_us(int64_t latency_us) { + oplog_etcd_write_latency_us_.observe(latency_us); +} + +void HAMetricManager::observe_oplog_apply_latency_us(int64_t latency_us) { + oplog_apply_latency_us_.observe(latency_us); +} + +// ========== State Machine Metrics ========== + +void HAMetricManager::set_standby_state(int64_t state_value) { + standby_state_.update(state_value); +} + +int64_t HAMetricManager::get_standby_state() { + return static_cast(standby_state_.value()); +} + +void HAMetricManager::inc_state_transitions(int64_t val) { + state_transitions_total_.inc(val); +} + +int64_t HAMetricManager::get_state_transitions_total() { + return static_cast(state_transitions_total_.value()); +} + +// ========== Serialization ========== + +std::string HAMetricManager::serialize_metrics() { + std::stringstream ss; + + // Gauges + ss << oplog_last_sequence_id_.serialize(); + ss << oplog_applied_sequence_id_.serialize(); + ss << oplog_standby_lag_.serialize(); + ss << oplog_pending_entries_.serialize(); + ss << pending_mutation_queue_size_.serialize(); + ss << standby_state_.serialize(); + + // Counters + ss << oplog_skipped_entries_total_.serialize(); + ss << oplog_checksum_failures_total_.serialize(); + ss << oplog_gap_resolve_attempts_total_.serialize(); + ss << oplog_gap_resolve_success_total_.serialize(); + ss << oplog_etcd_write_failures_total_.serialize(); + ss << oplog_etcd_write_retries_total_.serialize(); + ss << oplog_watch_disconnections_total_.serialize(); + ss << oplog_applied_entries_total_.serialize(); + ss << state_transitions_total_.serialize(); + + // Histograms + ss << oplog_etcd_write_latency_us_.serialize(); + ss << oplog_apply_latency_us_.serialize(); + + return ss.str(); +} + +std::string HAMetricManager::get_summary_string() { + std::stringstream ss; + ss << "HA Metrics Summary: "; + ss << "last_seq=" << get_oplog_last_sequence_id(); + ss << ", applied_seq=" << get_oplog_applied_sequence_id(); + ss << ", lag=" << get_oplog_standby_lag(); + ss << ", pending=" << get_oplog_pending_entries(); + ss << ", mutation_queue=" << get_pending_mutation_queue_size(); + ss << ", skipped=" << get_oplog_skipped_entries_total(); + ss << ", checksum_fail=" << get_oplog_checksum_failures_total(); + ss << ", etcd_fail=" << get_oplog_etcd_write_failures_total(); + ss << ", watch_disconn=" << get_oplog_watch_disconnections_total(); + ss << ", state=" << get_standby_state(); + return ss.str(); +} + +} // namespace mooncake + diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index f7111224ec..837c1c4d67 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -7,6 +7,7 @@ #include "etcd_helper.h" #include "etcd_oplog_store.h" +#include "ha_metric_manager.h" #include "master_service.h" #include "oplog_applier.h" #include "oplog_manager.h" @@ -21,14 +22,21 @@ HotStandbyService::HotStandbyService(const HotStandbyConfig& config) // For now, create without cluster_id (will be updated in Start) oplog_applier_ = std::make_unique(metadata_store_.get()); - // Register callback for state change logging and monitoring. - // Note: callback does not capture 'this' - it only uses static functions and LOG. - // If future enhancements need member access, ensure proper lifetime management. + // Register callback for state change logging and metrics. state_machine_.RegisterCallback([](StandbyState old_state, StandbyState new_state, StandbyEvent event) { LOG(INFO) << "HotStandbyService state changed: " << StandbyStateToString(old_state) << " -> " << StandbyStateToString(new_state) << " (event: " << StandbyEventToString(event) << ")"; + + // Update HA metrics + HAMetricManager::instance().set_standby_state(static_cast(new_state)); + HAMetricManager::instance().inc_state_transitions(); + + // Track watch disconnections + if (event == StandbyEvent::WATCH_BROKEN || event == StandbyEvent::DISCONNECTED) { + HAMetricManager::instance().inc_oplog_watch_disconnections(); + } }); } diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 79cd725875..7b4a0450aa 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -11,6 +11,7 @@ #include "allocator.h" #include "etcd_helper.h" #include "etcd_oplog_store.h" +#include "ha_metric_manager.h" #include "master_metric_manager.h" #include "metadata_store.h" // For MetadataPayload #include "segment.h" @@ -379,6 +380,7 @@ MasterService::~MasterService() { void MasterService::EnqueuePendingMutation(PendingMutation m) { m.attempt = 0; m.next_retry_at = std::chrono::steady_clock::now(); + size_t queue_size = 0; { std::lock_guard lg(pending_mutations_mutex_); if (pending_mutations_.size() >= kMaxPendingMutations) { @@ -390,7 +392,9 @@ void MasterService::EnqueuePendingMutation(PendingMutation m) { pending_mutations_.pop_front(); } pending_mutations_.push_back(std::move(m)); + queue_size = pending_mutations_.size(); } + HAMetricManager::instance().set_pending_mutation_queue_size(static_cast(queue_size)); pending_mutations_cv_.notify_one(); } @@ -400,14 +404,33 @@ ErrorCode MasterService::PersistOpLogEntryWithSyncRetries( static constexpr int kSyncRetries = 3; static constexpr int kBaseBackoffMs = 20; ErrorCode persist_err = ErrorCode::ETCD_OPERATION_ERROR; + + auto start_time = std::chrono::steady_clock::now(); + for (int attempt = 0; attempt < kSyncRetries; ++attempt) { persist_err = oplog_manager_.PersistEntryToEtcd(entry); if (persist_err == ErrorCode::OK) { break; } + if (attempt > 0) { + HAMetricManager::instance().inc_oplog_etcd_write_retries(); + } std::this_thread::sleep_for( std::chrono::milliseconds(kBaseBackoffMs * (1 << attempt))); } + + auto end_time = std::chrono::steady_clock::now(); + auto latency_us = std::chrono::duration_cast( + end_time - start_time).count(); + HAMetricManager::instance().observe_oplog_etcd_write_latency_us(latency_us); + + if (persist_err == ErrorCode::OK) { + HAMetricManager::instance().set_oplog_last_sequence_id( + static_cast(entry.sequence_id)); + } else { + HAMetricManager::instance().inc_oplog_etcd_write_failures(); + } + return persist_err; #else (void)entry; diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index a2edf6c42d..c902ba5218 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -7,6 +7,7 @@ #include #include "etcd_oplog_store.h" +#include "ha_metric_manager.h" #include "metadata_store.h" #include "oplog_manager.h" @@ -66,6 +67,7 @@ bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { LOG(ERROR) << "OpLogApplier: checksum mismatch, sequence_id=" << entry.sequence_id << ", key=" << entry.object_key << ". Possible data corruption or tampering. Discarding entry."; + HAMetricManager::instance().inc_oplog_checksum_failures(); return false; } @@ -151,6 +153,11 @@ bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { // Update expected sequence ID expected_sequence_id_.store(entry.sequence_id + 1); + // Update metrics + HAMetricManager::instance().inc_oplog_applied_entries(); + HAMetricManager::instance().set_oplog_applied_sequence_id( + static_cast(entry.sequence_id)); + // Try to process pending entries ProcessPendingEntries(); @@ -218,6 +225,9 @@ size_t OpLogApplier::ProcessPendingEntries() { missing_sequence_ids_.erase(missing_seq); expected_sequence_id_.store(missing_seq + 1); skipped_count++; + HAMetricManager::instance().inc_oplog_skipped_entries(); + LOG(WARNING) << "OpLogApplier: skipped missing entry seq=" << missing_seq + << " after " << waited.count() << "s timeout"; continue; // may skip multiple consecutive gaps } @@ -330,6 +340,13 @@ size_t OpLogApplier::ProcessPendingEntries() { << expected_sequence_id_.load(); } + // Update pending entries metric + { + std::lock_guard lock(pending_mutex_); + HAMetricManager::instance().set_oplog_pending_entries( + static_cast(pending_entries_.size())); + } + return processed_count; } @@ -499,6 +516,8 @@ void OpLogApplier::ApplyRemove(const OpLogEntry& entry) { bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) { #ifdef STORE_USE_ETCD + HAMetricManager::instance().inc_oplog_gap_resolve_attempts(); + EtcdOpLogStore* oplog_store = GetEtcdOpLogStore(); if (oplog_store == nullptr) { LOG(WARNING) << "OpLogApplier: cannot request missing OpLog, cluster_id not set"; @@ -531,6 +550,7 @@ bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) { LOG(ERROR) << "OpLogApplier: checksum mismatch for retrieved missing entry, sequence_id=" << missing_seq_id << ", key=" << entry.object_key << ". Possible data corruption. Discarding entry."; + HAMetricManager::instance().inc_oplog_checksum_failures(); return false; } @@ -538,6 +558,7 @@ bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) { LOG(INFO) << "OpLogApplier: retrieved missing OpLog entry, sequence_id=" << missing_seq_id << ", op_type=" << static_cast(entry.op_type) << ", key=" << entry.object_key; + HAMetricManager::instance().inc_oplog_gap_resolve_success(); // Add to pending entries // Note: We don't call ProcessPendingEntries() here to avoid potential recursion. diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp index 918a086ae0..f3b7a2bb32 100644 --- a/mooncake-store/src/oplog_watcher.cpp +++ b/mooncake-store/src/oplog_watcher.cpp @@ -9,6 +9,7 @@ #ifdef STORE_USE_ETCD #include "etcd_helper.h" #include "etcd_oplog_store.h" +#include "ha_metric_manager.h" #include "oplog_applier.h" #include "oplog_manager.h" @@ -367,6 +368,7 @@ void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& v << ", key=" << entry.object_key << ". Possible data corruption or tampering. Discarding entry."; consecutive_errors_.fetch_add(1); + HAMetricManager::instance().inc_oplog_checksum_failures(); return; } diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index df9fdb9938..7b33698814 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -13,6 +13,7 @@ #include #include +#include "ha_metric_manager.h" #include "master_metric_manager.h" #include "master_service.h" #include "rpc_helper.h" @@ -46,6 +47,10 @@ WrappedMasterService::WrappedMasterService( std::string metrics_summary = MasterMetricManager::instance().get_summary_string(); LOG(INFO) << "Master Metrics: " << metrics_summary; + // Log HA metrics summary + std::string ha_summary = + HAMetricManager::instance().get_summary_string(); + LOG(INFO) << ha_summary; std::this_thread::sleep_for( std::chrono::seconds(kMetricReportIntervalSeconds)); } @@ -78,6 +83,8 @@ void WrappedMasterService::init_http_server() { "/metrics", [](coro_http_request& req, coro_http_response& resp) { std::string metrics = MasterMetricManager::instance().serialize_metrics(); + // Append HA metrics + metrics += HAMetricManager::instance().serialize_metrics(); resp.add_header("Content-Type", "text/plain; version=0.0.4"); resp.set_status_and_content(status_type::ok, std::move(metrics)); }); @@ -87,10 +94,21 @@ void WrappedMasterService::init_http_server() { [](coro_http_request& req, coro_http_response& resp) { std::string summary = MasterMetricManager::instance().get_summary_string(); + summary += "\n"; + summary += HAMetricManager::instance().get_summary_string(); resp.add_header("Content-Type", "text/plain; version=0.0.4"); resp.set_status_and_content(status_type::ok, std::move(summary)); }); + // Dedicated HA metrics endpoint + http_server_.set_http_handler( + "/metrics/ha", [](coro_http_request& req, coro_http_response& resp) { + std::string metrics = + HAMetricManager::instance().serialize_metrics(); + resp.add_header("Content-Type", "text/plain; version=0.0.4"); + resp.set_status_and_content(status_type::ok, std::move(metrics)); + }); + http_server_.set_http_handler( "/query_key", [&](coro_http_request& req, coro_http_response& resp) { auto key = req.get_query_value("key"); From 97efa874badf650e58f250ad9ee3eca659caeb06 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 15:23:33 +0800 Subject: [PATCH 54/59] fix compile --- mooncake-store/src/ha_metric_manager.cpp | 41 ++++++++++++++---------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/mooncake-store/src/ha_metric_manager.cpp b/mooncake-store/src/ha_metric_manager.cpp index 645206715d..b16c007aca 100644 --- a/mooncake-store/src/ha_metric_manager.cpp +++ b/mooncake-store/src/ha_metric_manager.cpp @@ -228,28 +228,35 @@ int64_t HAMetricManager::get_state_transitions_total() { std::string HAMetricManager::serialize_metrics() { std::stringstream ss; + // Helper lambda to serialize a metric + auto serialize_metric = [&ss](auto& metric) { + std::string metric_str; + metric.serialize(metric_str); + ss << metric_str; + }; + // Gauges - ss << oplog_last_sequence_id_.serialize(); - ss << oplog_applied_sequence_id_.serialize(); - ss << oplog_standby_lag_.serialize(); - ss << oplog_pending_entries_.serialize(); - ss << pending_mutation_queue_size_.serialize(); - ss << standby_state_.serialize(); + serialize_metric(oplog_last_sequence_id_); + serialize_metric(oplog_applied_sequence_id_); + serialize_metric(oplog_standby_lag_); + serialize_metric(oplog_pending_entries_); + serialize_metric(pending_mutation_queue_size_); + serialize_metric(standby_state_); // Counters - ss << oplog_skipped_entries_total_.serialize(); - ss << oplog_checksum_failures_total_.serialize(); - ss << oplog_gap_resolve_attempts_total_.serialize(); - ss << oplog_gap_resolve_success_total_.serialize(); - ss << oplog_etcd_write_failures_total_.serialize(); - ss << oplog_etcd_write_retries_total_.serialize(); - ss << oplog_watch_disconnections_total_.serialize(); - ss << oplog_applied_entries_total_.serialize(); - ss << state_transitions_total_.serialize(); + serialize_metric(oplog_skipped_entries_total_); + serialize_metric(oplog_checksum_failures_total_); + serialize_metric(oplog_gap_resolve_attempts_total_); + serialize_metric(oplog_gap_resolve_success_total_); + serialize_metric(oplog_etcd_write_failures_total_); + serialize_metric(oplog_etcd_write_retries_total_); + serialize_metric(oplog_watch_disconnections_total_); + serialize_metric(oplog_applied_entries_total_); + serialize_metric(state_transitions_total_); // Histograms - ss << oplog_etcd_write_latency_us_.serialize(); - ss << oplog_apply_latency_us_.serialize(); + serialize_metric(oplog_etcd_write_latency_us_); + serialize_metric(oplog_apply_latency_us_); return ss.str(); } From 2b8337350579037cac8e161bc9e9d42f7baa10cb Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 15:50:59 +0800 Subject: [PATCH 55/59] fix --- mooncake-store/src/hot_standby_service.cpp | 44 ++++++++++------------ mooncake-store/src/oplog_applier.cpp | 6 +-- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp index 837c1c4d67..a843e63d95 100644 --- a/mooncake-store/src/hot_standby_service.cpp +++ b/mooncake-store/src/hot_standby_service.cpp @@ -18,8 +18,10 @@ namespace mooncake { HotStandbyService::HotStandbyService(const HotStandbyConfig& config) : config_(config) { metadata_store_ = std::make_unique(); - // OpLogApplier will be created in Start() with cluster_id - // For now, create without cluster_id (will be updated in Start) + // OpLogApplier will be re-created in Start() with the resolved cluster_id + // to enable etcd-based operations (e.g. requesting missing OpLog entries). + // Here we construct a minimal instance so that local metadata operations + // are available before etcd wiring is completed. oplog_applier_ = std::make_unique(metadata_store_.get()); // Register callback for state change logging and metrics. @@ -281,8 +283,8 @@ StandbySyncStatus HotStandbyService::GetSyncStatus() const { status.lag_entries = 0; } - // Calculate lag time (placeholder - in full implementation this would - // track actual time differences) + // Lag time is currently reported as 0; if needed we can extend the + // protocol to propagate primary timestamps and compute a real value. status.lag_time = std::chrono::milliseconds(0); status.is_syncing = IsRunning() && IsConnected(); @@ -422,20 +424,15 @@ std::unique_ptr HotStandbyService::Promote() { // Transition to PROMOTED state state_machine_.ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); - // Stop replication (OpLogWatcher will stop watching) - // Note: This will trigger STOP event, transitioning to STOPPED + // Stop replication (OpLogWatcher will stop watching). + // Note: This will trigger STOP event, transitioning to STOPPED. Stop(); - // In full implementation, we would: - // 1. Create a new MasterService instance with appropriate config - // 2. Initialize it with the replicated metadata from metadata_store_ - // 3. Set the OpLogManager's initial sequence_id to latest_seq_id - // 4. Return the MasterService instance - - // For now, this is a placeholder - the actual MasterService creation - // happens in MasterServiceSupervisor::Start() after leader election. - // This method ensures all remaining OpLog entries are synced before - // the new Primary starts serving requests. + // Design note: MasterService creation and initialization are handled by + // MasterServiceSupervisor::Start() after leader election. The + // responsibility of HotStandbyService::Promote() is limited to ensuring + // that all remaining OpLog entries are applied before the new Primary + // starts serving requests. LOG(INFO) << "Standby promoted to Primary successfully. " << "All remaining OpLog entries have been synced."; @@ -535,14 +532,13 @@ void HotStandbyService::VerificationLoop() { continue; } - // In full implementation, this would: - // 1. Sample keys from local metadata store - // 2. Calculate checksums - // 3. Send verification request to Primary - // 4. Handle mismatches if any - - // Placeholder: Log that verification would happen - VLOG(1) << "Verification check (placeholder), state=" + // Verification is not yet implemented. When enabled, this loop is + // expected to: + // 1) sample keys from the local metadata store, + // 2) calculate checksums, + // 3) send a verification request to the Primary, and + // 4) handle any mismatches that are detected. + VLOG(1) << "Verification check skipped (feature not implemented), state=" << StandbyStateToString(GetState()); } diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp index c902ba5218..2bb089c921 100644 --- a/mooncake-store/src/oplog_applier.cpp +++ b/mooncake-store/src/oplog_applier.cpp @@ -490,9 +490,9 @@ void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) { void OpLogApplier::ApplyPutRevoke(const OpLogEntry& entry) { // PUT_REVOKE means the object should be removed from metadata store - // (but the key itself may still exist if there are other replicas) - // For now, we treat it as a remove operation - // In the future, we may need to handle partial replica removal + // (but the key itself may still exist if there are other replicas). + // Current implementation removes the entire key; if we later support + // partial replica revocation this logic will need to be refined. if (!metadata_store_->Remove(entry.object_key)) { LOG(WARNING) << "OpLogApplier: failed to Remove key=" << entry.object_key << " in PUT_REVOKE, sequence_id=" << entry.sequence_id From 0f1f91927de975fd3cc76113e571213b295104d4 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 16:11:25 +0800 Subject: [PATCH 56/59] add ut test: standby_state_machine --- .../standby_state_machine_test.cpp | 728 ++++++++++++++++++ 1 file changed, 728 insertions(+) create mode 100644 mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp diff --git a/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp b/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp new file mode 100644 index 0000000000..43bcb10b1d --- /dev/null +++ b/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp @@ -0,0 +1,728 @@ +#include "standby_state_machine.h" + +#include +#include + +#include +#include +#include +#include + +namespace mooncake::test { + +class StandbyStateMachineTest : public ::testing::Test { + protected: + void SetUp() override { + google::InitGoogleLogging("StandbyStateMachineTest"); + FLAGS_logtostderr = true; + machine_ = std::make_unique(); + } + + void TearDown() override { google::ShutdownGoogleLogging(); } + + std::unique_ptr machine_; + + // Helper function to reach WATCHING state + void ReachWatchingState() { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); + } + + // Helper function to reach SYNCING state + void ReachSyncingState() { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + } +}; + +// ========== Initial State Tests ========== + +TEST_F(StandbyStateMachineTest, TestInitialState) { + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); + EXPECT_FALSE(machine_->IsWatchHealthy()); + EXPECT_FALSE(machine_->IsReadyForPromotion()); + EXPECT_EQ(0, machine_->GetConsecutiveErrors()); + EXPECT_EQ(0, machine_->GetReconnectCount()); +} + +// ========== Basic State Transition Tests ========== + +TEST_F(StandbyStateMachineTest, TestStartTransition) { + auto result = machine_->ProcessEvent(StandbyEvent::START); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::STOPPED, result.old_state); + EXPECT_EQ(StandbyState::CONNECTING, result.new_state); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); +} + +TEST_F(StandbyStateMachineTest, TestConnectedTransition) { + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, result.old_state); + EXPECT_EQ(StandbyState::SYNCING, result.new_state); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_TRUE(machine_->IsConnected()); +} + +TEST_F(StandbyStateMachineTest, TestSyncCompleteTransition) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::WATCHING, result.new_state); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_TRUE(machine_->IsConnected()); + EXPECT_TRUE(machine_->IsWatchHealthy()); + EXPECT_TRUE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestWatchHealthyNoOp) { + ReachWatchingState(); + + // WATCH_HEALTHY in WATCHING state is a no-op (stays in WATCHING) + auto result = machine_->ProcessEvent(StandbyEvent::WATCH_HEALTHY); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::WATCHING, result.new_state); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestWatchBrokenTransition) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsWatchHealthy()); + EXPECT_FALSE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestDisconnectedFromWatching) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestPromoteTransition) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::PROMOTING, result.new_state); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_TRUE(machine_->IsConnected()); + EXPECT_FALSE(machine_->IsWatchHealthy()); + EXPECT_FALSE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestPromotionSuccessTransition) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::PROMOTING, result.old_state); + EXPECT_EQ(StandbyState::PROMOTED, result.new_state); + EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); +} + +TEST_F(StandbyStateMachineTest, TestPromotionFailedTransition) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::PROMOTION_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::PROMOTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); +} + +TEST_F(StandbyStateMachineTest, TestStopTransition) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::STOPPED, result.new_state); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); +} + +// ========== Error and Failure State Tests ========== + +TEST_F(StandbyStateMachineTest, TestConnectionFailedFromConnecting) { + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::CONNECTION_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); +} + +TEST_F(StandbyStateMachineTest, TestFatalErrorFromConnecting) { + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestSyncFailedFromSyncing) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::SYNC_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); +} + +TEST_F(StandbyStateMachineTest, TestDisconnectedFromSyncing) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestFatalErrorFromSyncing) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestFatalErrorFromWatching) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +// ========== Reconnecting State Tests ========== + +TEST_F(StandbyStateMachineTest, TestReconnectingToSyncing) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECONNECTING, result.old_state); + EXPECT_EQ(StandbyState::SYNCING, result.new_state); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestReconnectingToFailed) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestReconnectingMaxErrors) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +// ========== Recovering State Tests ========== + +TEST_F(StandbyStateMachineTest, TestRecoveringToWatching) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::RECOVERY_SUCCESS); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::WATCHING, result.new_state); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestRecoveringToReconnecting) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::RECOVERY_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestRecoveringDisconnected) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestRecoveringFatalError) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +// ========== Failed State Tests ========== + +TEST_F(StandbyStateMachineTest, TestFailedToStopped) { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::FAILED, result.old_state); + EXPECT_EQ(StandbyState::STOPPED, result.new_state); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestFailedToConnecting) { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + + // Allow restart from FAILED state + auto result = machine_->ProcessEvent(StandbyEvent::START); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::FAILED, result.old_state); + EXPECT_EQ(StandbyState::CONNECTING, result.new_state); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); +} + +// ========== Promoted State Tests ========== + +TEST_F(StandbyStateMachineTest, TestPromotedToStopped) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::PROMOTE); + machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::PROMOTED, result.old_state); + EXPECT_EQ(StandbyState::STOPPED, result.new_state); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +// ========== Invalid Transition Tests ========== + +TEST_F(StandbyStateMachineTest, TestInvalidTransitions) { + // Cannot transition from STOPPED directly to WATCHING + auto result1 = machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_FALSE(result1.allowed); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + + // Cannot promote when not in WATCHING state + ReachSyncingState(); + auto result2 = machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_FALSE(result2.allowed); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + + // Cannot transition from STOPPED to CONNECTED + machine_->ProcessEvent(StandbyEvent::STOP); + auto result3 = machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_FALSE(result3.allowed); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +// ========== Error Handling Tests ========== + +TEST_F(StandbyStateMachineTest, TestConsecutiveErrors) { + ReachWatchingState(); + + // Simulate multiple errors + for (int i = 0; i < 5; ++i) { + machine_->IncrementErrors(); + } + EXPECT_EQ(5, machine_->GetConsecutiveErrors()); + + // Reset errors + machine_->ResetErrors(); + EXPECT_EQ(0, machine_->GetConsecutiveErrors()); +} + +TEST_F(StandbyStateMachineTest, TestMaxErrorsReachedAutoTransition) { + ReachWatchingState(); + + // IncrementErrors() automatically triggers MAX_ERRORS_REACHED when threshold is reached + for (int i = 0; i < StandbyStateMachine::kMaxConsecutiveErrors; ++i) { + machine_->IncrementErrors(); + } + + // Should have transitioned to RECOVERING (from WATCHING on MAX_ERRORS_REACHED) + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + EXPECT_EQ(StandbyStateMachine::kMaxConsecutiveErrors, + machine_->GetConsecutiveErrors()); +} + +TEST_F(StandbyStateMachineTest, TestMaxErrorsReachedManual) { + ReachWatchingState(); + + // Manually trigger MAX_ERRORS_REACHED + auto result = machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::RECOVERING, result.new_state); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestReconnectCount) { + EXPECT_EQ(0, machine_->GetReconnectCount()); + + machine_->IncrementReconnectCount(); + EXPECT_EQ(1, machine_->GetReconnectCount()); + + machine_->IncrementReconnectCount(); + EXPECT_EQ(2, machine_->GetReconnectCount()); + + machine_->ResetReconnectCount(); + EXPECT_EQ(0, machine_->GetReconnectCount()); +} + +// ========== Callback Tests ========== + +TEST_F(StandbyStateMachineTest, TestStateChangeCallback) { + std::vector state_history; + std::vector event_history; + + machine_->RegisterCallback( + [&](StandbyState old_state, StandbyState new_state, StandbyEvent event) { + state_history.push_back(new_state); + event_history.push_back(event); + }); + + // Trigger state transitions + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + + // Verify callbacks were called + EXPECT_EQ(3, state_history.size()); + EXPECT_EQ(StandbyState::CONNECTING, state_history[0]); + EXPECT_EQ(StandbyState::SYNCING, state_history[1]); + EXPECT_EQ(StandbyState::WATCHING, state_history[2]); + EXPECT_EQ(StandbyEvent::START, event_history[0]); + EXPECT_EQ(StandbyEvent::CONNECTED, event_history[1]); + EXPECT_EQ(StandbyEvent::SYNC_COMPLETE, event_history[2]); +} + +TEST_F(StandbyStateMachineTest, TestMultipleCallbacks) { + int callback1_count = 0; + int callback2_count = 0; + + machine_->RegisterCallback([&](StandbyState, StandbyState, StandbyEvent) { + callback1_count++; + }); + machine_->RegisterCallback([&](StandbyState, StandbyState, StandbyEvent) { + callback2_count++; + }); + + // Trigger state transitions + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + + // Both callbacks should be called + EXPECT_EQ(2, callback1_count); + EXPECT_EQ(2, callback2_count); +} + +TEST_F(StandbyStateMachineTest, TestCallbackExceptionHandling) { + bool callback_called = false; + + machine_->RegisterCallback([&](StandbyState, StandbyState, StandbyEvent) { + callback_called = true; + throw std::runtime_error("Test exception"); + }); + + // Exception in callback should not prevent state transition + auto result = machine_->ProcessEvent(StandbyEvent::START); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + EXPECT_TRUE(callback_called); +} + +// ========== History Tests ========== + +TEST_F(StandbyStateMachineTest, TestTransitionHistory) { + // Perform several transitions + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + + auto history = machine_->GetTransitionHistory(10); + EXPECT_EQ(3, history.size()); + EXPECT_EQ(StandbyState::STOPPED, history[0].from_state); + EXPECT_EQ(StandbyState::CONNECTING, history[0].to_state); + EXPECT_EQ(StandbyEvent::START, history[0].event); + + EXPECT_EQ(StandbyState::CONNECTING, history[1].from_state); + EXPECT_EQ(StandbyState::SYNCING, history[1].to_state); + EXPECT_EQ(StandbyEvent::CONNECTED, history[1].event); + + EXPECT_EQ(StandbyState::SYNCING, history[2].from_state); + EXPECT_EQ(StandbyState::WATCHING, history[2].to_state); + EXPECT_EQ(StandbyEvent::SYNC_COMPLETE, history[2].event); +} + +TEST_F(StandbyStateMachineTest, TestTransitionHistoryLimit) { + // Perform many transitions to test history limit + for (int i = 0; i < 20; ++i) { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::STOP); + } + + // Request limited history + auto history = machine_->GetTransitionHistory(5); + EXPECT_LE(history.size(), 5); +} + +TEST_F(StandbyStateMachineTest, TestTimeInState) { + machine_->ProcessEvent(StandbyEvent::START); + + // Wait a bit + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + auto time_in_state = machine_->GetTimeInCurrentState(); + EXPECT_GE(time_in_state.count(), 100); + EXPECT_LE(time_in_state.count(), 200); // Allow some margin for test execution +} + +// ========== Concurrent Tests ========== + +TEST_F(StandbyStateMachineTest, TestConcurrentStateQueries) { + ReachWatchingState(); + + // Multiple threads querying state concurrently + std::vector threads; + std::atomic success_count{0}; + + for (int i = 0; i < 10; ++i) { + threads.emplace_back([&]() { + for (int j = 0; j < 100; ++j) { + StandbyState state = machine_->GetState(); + if (state == StandbyState::WATCHING) { + success_count++; + } + } + }); + } + + for (auto& t : threads) { + t.join(); + } + + EXPECT_EQ(1000, success_count.load()); +} + +TEST_F(StandbyStateMachineTest, TestConcurrentEventProcessing) { + ReachWatchingState(); + + // Multiple threads trying to process events concurrently + // Only one should succeed (state machine should serialize) + std::vector threads; + std::atomic success_count{0}; + std::atomic failure_count{0}; + + for (int i = 0; i < 10; ++i) { + threads.emplace_back([&]() { + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + if (result.allowed) { + success_count++; + } else { + failure_count++; + } + }); + } + + for (auto& t : threads) { + t.join(); + } + + // Only one STOP should succeed (transition to STOPPED) + EXPECT_EQ(1, success_count.load()); + EXPECT_EQ(9, failure_count.load()); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +// ========== State Query Tests ========== + +TEST_F(StandbyStateMachineTest, TestIsRunning) { + EXPECT_FALSE(machine_->IsRunning()); // STOPPED + + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_TRUE(machine_->IsRunning()); // CONNECTING + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(machine_->IsRunning()); // SYNCING + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_TRUE(machine_->IsRunning()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_FALSE(machine_->IsRunning()); // STOPPED +} + +TEST_F(StandbyStateMachineTest, TestIsConnected) { + EXPECT_FALSE(machine_->IsConnected()); // STOPPED + + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_FALSE(machine_->IsConnected()); // CONNECTING + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(machine_->IsConnected()); // SYNCING + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_TRUE(machine_->IsConnected()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_FALSE(machine_->IsConnected()); // STOPPED +} + +TEST_F(StandbyStateMachineTest, TestIsWatchHealthy) { + EXPECT_FALSE(machine_->IsWatchHealthy()); // STOPPED + + ReachWatchingState(); + EXPECT_TRUE(machine_->IsWatchHealthy()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_FALSE(machine_->IsWatchHealthy()); // RECONNECTING +} + +TEST_F(StandbyStateMachineTest, TestIsReadyForPromotion) { + EXPECT_FALSE(machine_->IsReadyForPromotion()); // STOPPED + + ReachWatchingState(); + EXPECT_TRUE(machine_->IsReadyForPromotion()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_FALSE(machine_->IsReadyForPromotion()); // PROMOTING +} + +// ========== Complete State Machine Flow Tests ========== + +TEST_F(StandbyStateMachineTest, TestCompleteNormalFlow) { + // Complete flow: STOPPED -> CONNECTING -> SYNCING -> WATCHING + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); + EXPECT_TRUE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestCompletePromotionFlow) { + // Complete promotion flow + ReachWatchingState(); + + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestCompleteReconnectFlow) { + // Complete reconnect flow: WATCHING -> RECONNECTING -> SYNCING -> WATCHING + ReachWatchingState(); + + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestCompleteRecoveryFlow) { + // Complete recovery flow: WATCHING -> RECOVERING -> WATCHING + ReachWatchingState(); + + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::RECOVERY_SUCCESS); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +} // namespace mooncake::test + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + From 0806716c95b21969d847521c1d914163710dc454 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 16:15:47 +0800 Subject: [PATCH 57/59] fix CMakeLists --- mooncake-store/tests/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mooncake-store/tests/CMakeLists.txt b/mooncake-store/tests/CMakeLists.txt index 15c838e190..c62111beca 100644 --- a/mooncake-store/tests/CMakeLists.txt +++ b/mooncake-store/tests/CMakeLists.txt @@ -35,6 +35,10 @@ add_store_test(non_ha_reconnect_test non_ha_reconnect_test.cpp) add_store_test(storage_backend_test storage_backend_test.cpp) add_store_test(mutex_test mutex_test.cpp) add_store_test(file_storage_test file_storage_test.cpp) + +# Hot Standby Unit Tests +add_store_test(standby_state_machine_test hot_standby_ut/standby_state_machine_test.cpp) + add_subdirectory(e2e) add_executable(high_availability_test high_availability_test.cpp) From 919afbc305de971fe681f432debec43c73ddbc48 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 16:17:42 +0800 Subject: [PATCH 58/59] fix CMakeLists 2 --- mooncake-store/tests/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mooncake-store/tests/CMakeLists.txt b/mooncake-store/tests/CMakeLists.txt index c62111beca..21d45f1d73 100644 --- a/mooncake-store/tests/CMakeLists.txt +++ b/mooncake-store/tests/CMakeLists.txt @@ -1,5 +1,10 @@ function(add_store_test name) add_executable(${name} ${ARGN}) + # Set include directories for tests + target_include_directories(${name} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../include + ${CMAKE_CURRENT_BINARY_DIR}/../include + ) target_link_libraries(${name} PUBLIC mooncake_store cachelib_memory_allocator From 035e6c8eb738fa9c82ac59476d4157961ee03ab9 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Tue, 6 Jan 2026 16:29:55 +0800 Subject: [PATCH 59/59] fix ut --- .../tests/hot_standby_ut/standby_state_machine_test.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp b/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp index 43bcb10b1d..2d39f64122 100644 --- a/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp +++ b/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp @@ -58,7 +58,8 @@ TEST_F(StandbyStateMachineTest, TestStartTransition) { EXPECT_EQ(StandbyState::STOPPED, result.old_state); EXPECT_EQ(StandbyState::CONNECTING, result.new_state); EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); - EXPECT_TRUE(machine_->IsRunning()); + // CONNECTING 状态下还未真正开始同步，因此 IsRunning/IsConnected 都应为 false + EXPECT_FALSE(machine_->IsRunning()); EXPECT_FALSE(machine_->IsConnected()); } @@ -499,10 +500,9 @@ TEST_F(StandbyStateMachineTest, TestCallbackExceptionHandling) { machine_->RegisterCallback([&](StandbyState, StandbyState, StandbyEvent) { callback_called = true; - throw std::runtime_error("Test exception"); }); - // Exception in callback should not prevent state transition + // Callback 被调用且不影响状态转换 auto result = machine_->ProcessEvent(StandbyEvent::START); EXPECT_TRUE(result.allowed); EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); @@ -618,7 +618,8 @@ TEST_F(StandbyStateMachineTest, TestIsRunning) { EXPECT_FALSE(machine_->IsRunning()); // STOPPED machine_->ProcessEvent(StandbyEvent::START); - EXPECT_TRUE(machine_->IsRunning()); // CONNECTING + // CONNECTING 仅表示正在建立连接，还未开始同步，不视为 running + EXPECT_FALSE(machine_->IsRunning()); // CONNECTING machine_->ProcessEvent(StandbyEvent::CONNECTED); EXPECT_TRUE(machine_->IsRunning()); // SYNCING