diff --git a/dependencies.sh b/dependencies.sh index 80f1c3ba69..ebcafe4221 100755 --- a/dependencies.sh +++ b/dependencies.sh @@ -121,6 +121,7 @@ SYSTEM_PACKAGES="build-essential \ libcurl4-openssl-dev \ libhiredis-dev \ libjemalloc-dev \ + libxxhash-dev \ pkg-config \ patchelf" diff --git a/mooncake-common/etcd/etcd_wrapper.go b/mooncake-common/etcd/etcd_wrapper.go index e57ae53628..ec22d98636 100644 --- a/mooncake-common/etcd/etcd_wrapper.go +++ b/mooncake-common/etcd/etcd_wrapper.go @@ -9,9 +9,11 @@ import "C" import ( "context" + "encoding/json" "strings" "sync" "time" + "unsafe" clientv3 "go.etcd.io/etcd/client/v3" ) @@ -20,18 +22,21 @@ import ( // and can be configured separately. var ( // etcd client for transform engine - globalClient *clientv3.Client - globalMutex sync.Mutex - globalRefCount int + globalClient *clientv3.Client + globalMutex sync.Mutex + globalRefCount int // etcd client for store - storeClient *clientv3.Client - storeMutex sync.Mutex + storeClient *clientv3.Client + storeMutex sync.Mutex // keep alive contexts for store - storeKeepAliveCtx = make(map[int64]context.CancelFunc) - storeKeepAliveMutex sync.Mutex + storeKeepAliveCtx = make(map[int64]context.CancelFunc) + storeKeepAliveMutex sync.Mutex // watch contexts for store - storeWatchCtx = make(map[string]context.CancelFunc) - storeWatchMutex sync.Mutex + storeWatchCtx = make(map[string]context.CancelFunc) + storeWatchMutex sync.Mutex + // watch contexts for prefix watch + storePrefixWatchCtx = make(map[string]context.CancelFunc) + storePrefixWatchMutex sync.Mutex ) //export NewEtcdClient @@ -43,30 +48,30 @@ func NewEtcdClient(endpoints *C.char, errMsg **C.char) int { return 0 } - MaxMsgSize := 32*1024*1024 - endpointStr := C.GoString(endpoints) - // Support multiple endpoints separated by comma or semicolon - // Normalize separators to semicolon first, then split - endpointStr = strings.ReplaceAll(endpointStr, ",", ";") - parts := strings.Split(endpointStr, ";") - var validEndpoints []string - for _, ep := range parts { - ep = strings.TrimSpace(ep) - if ep != "" { - validEndpoints = append(validEndpoints, ep) - } - } - if len(validEndpoints) == 0 { - *errMsg = C.CString("no valid endpoints provided") - return -1 - } - - cli, err := clientv3.New(clientv3.Config{ - Endpoints: validEndpoints, - DialTimeout: 5 * time.Second, - MaxCallSendMsgSize: MaxMsgSize, - MaxCallRecvMsgSize: MaxMsgSize, - }) + MaxMsgSize := 32 * 1024 * 1024 + endpointStr := C.GoString(endpoints) + // Support multiple endpoints separated by comma or semicolon + // Normalize separators to semicolon first, then split + endpointStr = strings.ReplaceAll(endpointStr, ",", ";") + parts := strings.Split(endpointStr, ";") + var validEndpoints []string + for _, ep := range parts { + ep = strings.TrimSpace(ep) + if ep != "" { + validEndpoints = append(validEndpoints, ep) + } + } + if len(validEndpoints) == 0 { + *errMsg = C.CString("no valid endpoints provided") + return -1 + } + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: validEndpoints, + DialTimeout: 5 * time.Second, + MaxCallSendMsgSize: MaxMsgSize, + MaxCallRecvMsgSize: MaxMsgSize, + }) if err != nil { *errMsg = C.CString(err.Error()) @@ -159,11 +164,14 @@ func NewStoreEtcdClient(endpoints *C.char, errMsg **C.char) int { } endpointStr := C.GoString(endpoints) + // Support multiple endpoints separated by comma or semicolon. + endpointStr = strings.ReplaceAll(endpointStr, ",", ";") endpointList := strings.Split(endpointStr, ";") - + // Filter out any empty strings that might result from splitting var validEndpoints []string for _, ep := range endpointList { + ep = strings.TrimSpace(ep) if ep != "" { validEndpoints = append(validEndpoints, ep) } @@ -235,37 +243,37 @@ func EtcdStoreGrantLeaseWrapper(ttl int64, leaseId *int64, errMsg **C.char) int //export EtcdStoreCreateWithLeaseWrapper func EtcdStoreCreateWithLeaseWrapper(key *C.char, keySize C.int, value *C.char, valueSize C.int, leaseId int64, revisionId *int64, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - k := C.GoStringN(key, keySize) - v := C.GoStringN(value, valueSize) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - // Create a transaction - txn := storeClient.Txn(ctx) - - // Only put the key if it does not exist - resp, err := txn.If(clientv3.Compare(clientv3.CreateRevision(k), "=", 0)). - Then(clientv3.OpPut(k, v, clientv3.WithLease(clientv3.LeaseID(leaseId)))). - Commit() - - if err != nil { - *errMsg = C.CString(err.Error()) - return -1 - } - - // If the key already existed, resp.Succeeded will be false - // If we created the key, resp.Succeeded will be true - if resp.Succeeded { - *revisionId = resp.Header.Revision - return 0; - } else { - *errMsg = C.CString("etcd transaction failed") - return -2 - } + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + v := C.GoStringN(value, valueSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Create a transaction + txn := storeClient.Txn(ctx) + + // Only put the key if it does not exist + resp, err := txn.If(clientv3.Compare(clientv3.CreateRevision(k), "=", 0)). + Then(clientv3.OpPut(k, v, clientv3.WithLease(clientv3.LeaseID(leaseId)))). + Commit() + + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + // If the key already existed, resp.Succeeded will be false + // If we created the key, resp.Succeeded will be true + if resp.Succeeded { + *revisionId = resp.Header.Revision + return 0 + } else { + *errMsg = C.CString("etcd transaction failed") + return -2 + } } /* @@ -274,77 +282,77 @@ func EtcdStoreCreateWithLeaseWrapper(key *C.char, keySize C.int, value *C.char, * other than the one we want to delete. In that case, that context will * be deleted before being cancelled and will not be able to be cancelled * anymore. -*/ + */ func cancelAndDeleteWatch(k string) int { - storeWatchMutex.Lock() - defer storeWatchMutex.Unlock() - - if cancel, exists := storeWatchCtx[k]; exists { - cancel() - delete(storeWatchCtx, k) - return 0 - } + storeWatchMutex.Lock() + defer storeWatchMutex.Unlock() + + if cancel, exists := storeWatchCtx[k]; exists { + cancel() + delete(storeWatchCtx, k) + return 0 + } return -1 } //export EtcdStoreWatchUntilDeletedWrapper func EtcdStoreWatchUntilDeletedWrapper(key *C.char, keySize C.int, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - k := C.GoStringN(key, keySize) - - // Create a context with cancel function - ctx, cancel := context.WithCancel(context.Background()) - - // Store the cancel function - storeWatchMutex.Lock() - if _, exists := storeWatchCtx[k]; exists { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storeWatchMutex.Lock() + if _, exists := storeWatchCtx[k]; exists { storeWatchMutex.Unlock() - *errMsg = C.CString("This key is already being watched") - return -1 - } - storeWatchCtx[k] = cancel - storeWatchMutex.Unlock() + *errMsg = C.CString("This key is already being watched") + return -1 + } + storeWatchCtx[k] = cancel + storeWatchMutex.Unlock() // Make sure to delete from the map before returning defer cancelAndDeleteWatch(k) - // Start watching the key - watchChan := storeClient.Watch(ctx, k) - - // Wait for the key to be deleted - for { - select { - case watchResp, ok := <-watchChan: - if !ok { - // Channel closed unexpectedly - *errMsg = C.CString("watch channel closed unexpectedly") - return -1 - } - for _, event := range watchResp.Events { - if event.Type == clientv3.EventTypeDelete { - // Clean up the context when done - return 0 - } - } - case <-ctx.Done(): - // Context was cancelled + // Start watching the key + watchChan := storeClient.Watch(ctx, k) + + // Wait for the key to be deleted + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + // Channel closed unexpectedly + *errMsg = C.CString("watch channel closed unexpectedly") + return -1 + } + for _, event := range watchResp.Events { + if event.Type == clientv3.EventTypeDelete { + // Clean up the context when done + return 0 + } + } + case <-ctx.Done(): + // Context was cancelled *errMsg = C.CString("watch context cancelled") - return -2 - } - } + return -2 + } + } } //export EtcdStoreCancelWatchWrapper func EtcdStoreCancelWatchWrapper(key *C.char, keySize C.int, errMsg **C.char) int { - k := C.GoStringN(key, keySize) - if cancelAndDeleteWatch(k) == -1 { - *errMsg = C.CString("no watch context found for the given key") - return -1 - } - return 0 + k := C.GoStringN(key, keySize) + if cancelAndDeleteWatch(k) == -1 { + *errMsg = C.CString("no watch context found for the given key") + return -1 + } + return 0 } /* @@ -353,76 +361,588 @@ func EtcdStoreCancelWatchWrapper(key *C.char, keySize C.int, errMsg **C.char) in * other than the one we want to delete. In that case, that context will * be deleted before being cancelled and will not be able to be cancelled * anymore. -*/ + */ func cancelAndDeleteKeepAlive(leaseId int64) int { - storeKeepAliveMutex.Lock() - defer storeKeepAliveMutex.Unlock() - - if cancel, exists := storeKeepAliveCtx[leaseId]; exists { - cancel() - delete(storeKeepAliveCtx, leaseId) - return 0 - } + storeKeepAliveMutex.Lock() + defer storeKeepAliveMutex.Unlock() + + if cancel, exists := storeKeepAliveCtx[leaseId]; exists { + cancel() + delete(storeKeepAliveCtx, leaseId) + return 0 + } return -1 } //export EtcdStoreKeepAliveWrapper func EtcdStoreKeepAliveWrapper(leaseId int64, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - - // Create a context with cancel function - ctx, cancel := context.WithCancel(context.Background()) - - // Store the cancel function - storeKeepAliveMutex.Lock() + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storeKeepAliveMutex.Lock() if _, exists := storeKeepAliveCtx[leaseId]; exists { storeKeepAliveMutex.Unlock() - *errMsg = C.CString("This lease id is already being kept alive") - return -1 - } - storeKeepAliveCtx[leaseId] = cancel - storeKeepAliveMutex.Unlock() + *errMsg = C.CString("This lease id is already being kept alive") + return -1 + } + storeKeepAliveCtx[leaseId] = cancel + storeKeepAliveMutex.Unlock() // Make sure to delete from the map before returning - defer cancelAndDeleteKeepAlive(leaseId) - - // Start keep alive - keepAliveChan, err := storeClient.KeepAlive(ctx, clientv3.LeaseID(leaseId)) - if err != nil { - *errMsg = C.CString(err.Error()) - return -1 - } - - // Wait for keep alive responses - for { - select { - case resp, ok := <-keepAliveChan: - if !ok { - *errMsg = C.CString("keep alive channel closed") - return -1 - } - if resp == nil { - *errMsg = C.CString("keep alive response is nil") - return -1 - } - // Keep alive successful, continue - case <-ctx.Done(): + defer cancelAndDeleteKeepAlive(leaseId) + + // Start keep alive + keepAliveChan, err := storeClient.KeepAlive(ctx, clientv3.LeaseID(leaseId)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + // Wait for keep alive responses + for { + select { + case resp, ok := <-keepAliveChan: + if !ok { + *errMsg = C.CString("keep alive channel closed") + return -1 + } + if resp == nil { + *errMsg = C.CString("keep alive response is nil") + return -1 + } + // Keep alive successful, continue + case <-ctx.Done(): // Context cancelled *errMsg = C.CString("keep alive context cancelled") - return -2 - } - } + return -2 + } + } } //export EtcdStoreCancelKeepAliveWrapper func EtcdStoreCancelKeepAliveWrapper(leaseId int64, errMsg **C.char) int { - if cancelAndDeleteKeepAlive(leaseId) == -1 { - *errMsg = C.CString("no keep alive context found for the given lease ID") - return -1 - } - return 0 + if cancelAndDeleteKeepAlive(leaseId) == -1 { + *errMsg = C.CString("no keep alive context found for the given lease ID") + return -1 + } + return 0 +} + +//export EtcdStorePutWrapper +func EtcdStorePutWrapper(key *C.char, keySize C.int, value *C.char, valueSize C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + v := C.GoStringN(value, valueSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := storeClient.Put(ctx, k, v) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + return 0 +} + +// Create key if absent (CAS on CreateRevision==0). +// Return: +// - 0 on success +// - -2 if key already exists +// - -1 on error +// +//export EtcdStoreCreateWrapper +func EtcdStoreCreateWrapper(key *C.char, keySize C.int, value *C.char, valueSize C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + v := C.GoStringN(value, valueSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + txn := storeClient.Txn(ctx) + resp, err := txn.If(clientv3.Compare(clientv3.CreateRevision(k), "=", 0)). + Then(clientv3.OpPut(k, v)). + Commit() + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + if resp.Succeeded { + return 0 + } + *errMsg = C.CString("key already exists") + return -2 +} + +//export EtcdStoreGetWithPrefixWrapper +func EtcdStoreGetWithPrefixWrapper(prefix *C.char, prefixSize C.int, keys **C.char, keySizes **C.int, values **C.char, valueSizes **C.int, count *C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := storeClient.Get(ctx, p, clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + if len(resp.Kvs) == 0 { + *count = 0 + return 0 + } + + // Allocate arrays for keys and values + keyCount := len(resp.Kvs) + *count = C.int(keyCount) + + // Allocate memory for arrays + keysArray := (*[1 << 30]*C.char)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof((*C.char)(nil))))) + keySizesArray := (*[1 << 30]C.int)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof(C.int(0))))) + valuesArray := (*[1 << 30]*C.char)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof((*C.char)(nil))))) + valueSizesArray := (*[1 << 30]C.int)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof(C.int(0))))) + + for i, kv := range resp.Kvs { + keysArray[i] = C.CString(string(kv.Key)) + keySizesArray[i] = C.int(len(kv.Key)) + valuesArray[i] = C.CString(string(kv.Value)) + valueSizesArray[i] = C.int(len(kv.Value)) + } + + *keys = (*C.char)(unsafe.Pointer(keysArray)) + *keySizes = (*C.int)(unsafe.Pointer(keySizesArray)) + *values = (*C.char)(unsafe.Pointer(valuesArray)) + *valueSizes = (*C.int)(unsafe.Pointer(valueSizesArray)) + + return 0 +} + +//export EtcdStoreGetRangeAsJsonWrapper +func EtcdStoreGetRangeAsJsonWrapper(startKey *C.char, startKeySize C.int, endKey *C.char, endKeySize C.int, limit C.int, outJson **C.char, outJsonSize *C.int, revisionId *C.longlong, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + start := C.GoStringN(startKey, startKeySize) + end := C.GoStringN(endKey, endKeySize) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + opts := []clientv3.OpOption{ + clientv3.WithRange(end), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), + } + if limit > 0 { + opts = append(opts, clientv3.WithLimit(int64(limit))) + } + resp, err := storeClient.Get(ctx, start, opts...) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + if resp != nil && resp.Header != nil { + *revisionId = C.longlong(resp.Header.Revision) + } else { + *revisionId = 0 + } + + type kvPair struct { + Key string `json:"key"` + Value string `json:"value"` + } + kvs := make([]kvPair, 0, len(resp.Kvs)) + for _, kv := range resp.Kvs { + kvs = append(kvs, kvPair{Key: string(kv.Key), Value: string(kv.Value)}) + } + b, jerr := json.Marshal(kvs) + if jerr != nil { + *errMsg = C.CString(jerr.Error()) + return -1 + } + + *outJson = C.CString(string(b)) + *outJsonSize = C.int(len(b)) + return 0 +} + +//export EtcdStoreGetFirstKeyWithPrefixWrapper +func EtcdStoreGetFirstKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, firstKey **C.char, firstKeySize *C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + resp, err := storeClient.Get(ctx, p, clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), clientv3.WithLimit(1)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + if len(resp.Kvs) == 0 { + *errMsg = C.CString("no key found with prefix") + return -2 + } + kv := resp.Kvs[0] + *firstKey = C.CString(string(kv.Key)) + *firstKeySize = C.int(len(kv.Key)) + return 0 +} + +//export EtcdStoreGetLastKeyWithPrefixWrapper +func EtcdStoreGetLastKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, lastKey **C.char, lastKeySize *C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + resp, err := storeClient.Get( + ctx, p, + clientv3.WithPrefix(), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortDescend), + clientv3.WithLimit(1), + ) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + if len(resp.Kvs) == 0 { + *errMsg = C.CString("no key found with prefix") + return -2 + } + kv := resp.Kvs[0] + *lastKey = C.CString(string(kv.Key)) + *lastKeySize = C.int(len(kv.Key)) + return 0 +} + +//export EtcdStoreDeleteRangeWrapper +func EtcdStoreDeleteRangeWrapper(startKey *C.char, startKeySize C.int, endKey *C.char, endKeySize C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + start := C.GoStringN(startKey, startKeySize) + end := C.GoStringN(endKey, endKeySize) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := storeClient.Delete(ctx, start, clientv3.WithRange(end)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + return 0 +} + +//export EtcdStoreWatchWithPrefixWrapper +func EtcdStoreWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, callbackContext unsafe.Pointer, callbackFunc unsafe.Pointer, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + if callbackFunc == nil { + *errMsg = C.CString("callback function is nil") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storePrefixWatchMutex.Lock() + if _, exists := storePrefixWatchCtx[p]; exists { + storePrefixWatchMutex.Unlock() + *errMsg = C.CString("This prefix is already being watched") + return -1 + } + storePrefixWatchCtx[p] = cancel + storePrefixWatchMutex.Unlock() + + // Start watching in a goroutine + go func() { + defer cancelAndDeletePrefixWatch(p) + + // Start watching the prefix + watchChan := storeClient.Watch(ctx, p, clientv3.WithPrefix()) + + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + // Channel closed unexpectedly + return + } + if watchResp.Err() != nil { + // Watch error, stop watching + return + } + + // Process each event + for _, event := range watchResp.Events { + keyStr := string(event.Kv.Key) + keyPtr := C.CString(keyStr) + keySize := C.size_t(len(keyStr)) + + var valuePtr *C.char + var valueSize C.size_t + var eventType C.int + + if event.Type == clientv3.EventTypePut { + eventType = C.int(0) // WatchEventTypePut + valueStr := string(event.Kv.Value) + valuePtr = C.CString(valueStr) + valueSize = C.size_t(len(valueStr)) + } else if event.Type == clientv3.EventTypeDelete { + eventType = C.int(1) // WatchEventTypeDelete + valuePtr = nil + valueSize = 0 + } + + // Call the C callback function + // Convert unsafe.Pointer to function pointer type and call it + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int))(callbackFunc) + (*callbackType)(callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType) + + // Free the C strings + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + } + case <-ctx.Done(): + // Context was cancelled + return + } + } + }() + + return 0 +} + +//export EtcdStoreWatchWithPrefixFromRevisionWrapper +func EtcdStoreWatchWithPrefixFromRevisionWrapper(prefix *C.char, prefixSize C.int, startRevision C.longlong, callbackContext unsafe.Pointer, callbackFunc unsafe.Pointer, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + if callbackFunc == nil { + *errMsg = C.CString("callback function is nil") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storePrefixWatchMutex.Lock() + if _, exists := storePrefixWatchCtx[p]; exists { + storePrefixWatchMutex.Unlock() + *errMsg = C.CString("This prefix is already being watched") + return -1 + } + storePrefixWatchCtx[p] = cancel + storePrefixWatchMutex.Unlock() + + go func() { + defer cancelAndDeletePrefixWatch(p) + + opts := []clientv3.OpOption{clientv3.WithPrefix()} + if startRevision > 0 { + opts = append(opts, clientv3.WithRev(int64(startRevision))) + } + watchChan := storeClient.Watch(ctx, p, opts...) + + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + return + } + if watchResp.Err() != nil { + return + } + + for _, event := range watchResp.Events { + keyStr := string(event.Kv.Key) + keyPtr := C.CString(keyStr) + keySize := C.size_t(len(keyStr)) + + var valuePtr *C.char + var valueSize C.size_t + var eventType C.int + + if event.Type == clientv3.EventTypePut { + eventType = C.int(0) + valueStr := string(event.Kv.Value) + valuePtr = C.CString(valueStr) + valueSize = C.size_t(len(valueStr)) + } else if event.Type == clientv3.EventTypeDelete { + eventType = C.int(1) + valuePtr = nil + valueSize = 0 + } + + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int))(callbackFunc) + (*callbackType)(callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType) + + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + } + case <-ctx.Done(): + return + } + } + }() + + return 0 +} + +//export EtcdStoreWatchWithPrefixFromRevisionV2Wrapper +func EtcdStoreWatchWithPrefixFromRevisionV2Wrapper(prefix *C.char, prefixSize C.int, startRevision C.longlong, callbackContext unsafe.Pointer, callbackFunc unsafe.Pointer, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + if callbackFunc == nil { + *errMsg = C.CString("callback function is nil") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + + ctx, cancel := context.WithCancel(context.Background()) + + storePrefixWatchMutex.Lock() + if _, exists := storePrefixWatchCtx[p]; exists { + storePrefixWatchMutex.Unlock() + *errMsg = C.CString("This prefix is already being watched") + return -1 + } + storePrefixWatchCtx[p] = cancel + storePrefixWatchMutex.Unlock() + + go func() { + defer cancelAndDeletePrefixWatch(p) + + opts := []clientv3.OpOption{clientv3.WithPrefix()} + if startRevision > 0 { + opts = append(opts, clientv3.WithRev(int64(startRevision))) + } + watchChan := storeClient.Watch(ctx, p, opts...) + + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + // Channel closed unexpectedly. Notify C++ watcher to reconnect. + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int, C.longlong))(callbackFunc) + (*callbackType)(callbackContext, nil, 0, nil, 0, C.int(2) /*WATCH_BROKEN*/, C.longlong(0)) + return + } + if watchResp.Err() != nil { + // Watch error, stop watching. Notify C++ watcher to reconnect. + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int, C.longlong))(callbackFunc) + (*callbackType)(callbackContext, nil, 0, nil, 0, C.int(2) /*WATCH_BROKEN*/, C.longlong(0)) + return + } + + // Use response-level revision as a more stable resume point. + // (It can be >= individual event's ModRevision.) + // Note: watchResp.Header is a value type, not a pointer, so we can directly access it. + respRev := int64(0) + if watchResp.Header.Revision > 0 { + respRev = watchResp.Header.Revision + } + + for _, event := range watchResp.Events { + keyStr := string(event.Kv.Key) + keyPtr := C.CString(keyStr) + keySize := C.size_t(len(keyStr)) + + var valuePtr *C.char + var valueSize C.size_t + var eventType C.int + + if event.Type == clientv3.EventTypePut { + eventType = C.int(0) + valueStr := string(event.Kv.Value) + valuePtr = C.CString(valueStr) + valueSize = C.size_t(len(valueStr)) + } else if event.Type == clientv3.EventTypeDelete { + eventType = C.int(1) + valuePtr = nil + valueSize = 0 + } + + modRev := C.longlong(0) + if event.Kv != nil { + evRev := event.Kv.ModRevision + if respRev > evRev { + evRev = respRev + } + modRev = C.longlong(evRev) + } else if respRev > 0 { + modRev = C.longlong(respRev) + } + + // Callback signature: + // void cb(void* ctx, char* key, size_t keySize, char* value, size_t valueSize, int eventType, long long modRev) + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int, C.longlong))(callbackFunc) + (*callbackType)(callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType, modRev) + + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + } + case <-ctx.Done(): + return + } + } + }() + + return 0 +} + +func cancelAndDeletePrefixWatch(p string) int { + storePrefixWatchMutex.Lock() + defer storePrefixWatchMutex.Unlock() + + if cancel, exists := storePrefixWatchCtx[p]; exists { + cancel() + delete(storePrefixWatchCtx, p) + return 0 + } + return -1 +} + +//export EtcdStoreCancelWatchWithPrefixWrapper +func EtcdStoreCancelWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, errMsg **C.char) int { + p := C.GoStringN(prefix, prefixSize) + if cancelAndDeletePrefixWatch(p) == -1 { + *errMsg = C.CString("no watch context found for the given prefix") + return -1 + } + return 0 } func main() {} diff --git a/mooncake-store/include/etcd_helper.h b/mooncake-store/include/etcd_helper.h index 1f272142ac..38e48743f8 100644 --- a/mooncake-store/include/etcd_helper.h +++ b/mooncake-store/include/etcd_helper.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include "types.h" @@ -90,6 +92,133 @@ class EtcdHelper { */ static ErrorCode CancelKeepAlive(EtcdLeaseId lease_id); + /* + * @brief Put a key-value pair to etcd. + * @param key: The key to put. + * @param key_size: The size of the key in bytes. + * @param value: The value to put. + * @param value_size: The size of the value in bytes. + * @return: Error code. + */ + static ErrorCode Put(const char* key, const size_t key_size, + const char* value, const size_t value_size); + + /* + * @brief Create a key-value pair in etcd if the key does not already exist. + * This is implemented via etcd transaction (CreateRevision == 0). + * @return: OK on success; ETCD_TRANSACTION_FAIL if key already exists. + */ + static ErrorCode Create(const char* key, const size_t key_size, + const char* value, const size_t value_size); + + /* + * @brief Get all key-value pairs with a given prefix. + * @param prefix: The prefix to search for. + * @param prefix_size: The size of the prefix in bytes. + * @param keys: Output param, vector of keys. + * @param values: Output param, vector of values. + * @return: Error code. + */ + static ErrorCode GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values); + + /* + * @brief Range get in etcd and return result as a JSON array string. + * This avoids complex cross-language memory management for key/value arrays. + * @param start_key: Start key (inclusive). + * @param start_key_size: Size in bytes. + * @param end_key: End key (exclusive). + * @param end_key_size: Size in bytes. + * @param limit: Maximum number of kvs to return (0 means no limit). + * @param json: Output JSON string, format: [{"key":"...","value":"..."}] + * @param revision_id: Output etcd revision of this read (resp.Header.Revision). + */ + static ErrorCode GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id); + + /* + * @brief Get the first key with a given prefix (sorted by key). + * @param prefix: The prefix to search for. + * @param prefix_size: The size of the prefix in bytes. + * @param first_key: Output param, the first key found. + * @return: Error code. ETCD_KEY_NOT_EXIST if no key found. + */ + static ErrorCode GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key); + + /* + * @brief Get the last key with a given prefix (sorted by key descending). + * @param prefix: The prefix to search for. + * @param prefix_size: The size of the prefix in bytes. + * @param last_key: Output param, the last key found. + * @return: Error code. ETCD_KEY_NOT_EXIST if no key found. + */ + static ErrorCode GetLastKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& last_key); + + /* + * @brief Delete a range of keys from etcd. + * @param start_key: The start key (inclusive). + * @param start_key_size: The size of the start key in bytes. + * @param end_key: The end key (exclusive). + * @param end_key_size: The size of the end key in bytes. + * @return: Error code. + */ + static ErrorCode DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size); + + /* + * @brief Watch all keys with a given prefix for changes. + * This is a non-blocking function that starts watching in a background + * goroutine. Events are delivered via the callback function. + * @param prefix: The prefix to watch. + * @param prefix_size: The size of the prefix in bytes. + * @param callback_context: User context passed to the callback function. + * @param callback_func: Callback function called for each watch event. + * Signature: void callback(void* context, const char* key, size_t key_size, + * const char* value, size_t value_size, int event_type) + * event_type: 0 = PUT, 1 = DELETE + * @return: Error code. + */ + static ErrorCode WatchWithPrefix(const char* prefix, const size_t prefix_size, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, + const char*, size_t, int)); + + /* + * @brief Watch all keys with a given prefix from a specific etcd revision. + * Callback includes `mod_revision` for precise resume. + * (Implementation may pass max(event.ModRevision, watchResp.Header.Revision).) + * @param callback_func: void cb(void* ctx, const char* key, size_t key_size, + * const char* value, size_t value_size, + * int event_type, int64_t mod_revision) + * event_type: 0=PUT, 1=DELETE, 2=WATCH_BROKEN (watch ended; reconnect) + */ + static ErrorCode WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)); + + /* + * @brief Cancel watching a prefix. + * @param prefix: The prefix to stop watching. + * @param prefix_size: The size of the prefix in bytes. + * @return: Error code. + */ + static ErrorCode CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size); + private: // Variables that are used to ensure the etcd client // is only connected once. diff --git a/mooncake-store/include/etcd_oplog_store.h b/mooncake-store/include/etcd_oplog_store.h new file mode 100644 index 0000000000..013c38eea3 --- /dev/null +++ b/mooncake-store/include/etcd_oplog_store.h @@ -0,0 +1,202 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "types.h" + +namespace mooncake { + +/** + * @brief Store for OpLog entries in etcd. + * + * This class is responsible for writing OpLog entries to etcd and reading them back. + * OpLog entries are stored with keys in the format: + * /oplog/{cluster_id}/{sequence_id} + * + * The latest sequence_id is also stored at: + * /oplog/{cluster_id}/latest + */ +class EtcdOpLogStore { + public: + /** + * @brief Constructor. + * @param cluster_id: The cluster ID for this OpLog store. + * @param enable_latest_seq_batch_update: Whether to start background thread + * to batch-update `/latest`. Readers (Standby) should set this to false + * to avoid unnecessary thread creation. + */ + explicit EtcdOpLogStore(const std::string& cluster_id, + bool enable_latest_seq_batch_update = false); + + /** + * @brief Write an OpLog entry to etcd. + * @param entry: The OpLog entry to write. + * @return: Error code. + */ + ErrorCode WriteOpLog(const OpLogEntry& entry); + + /** + * @brief Read an OpLog entry from etcd by sequence_id. + * @param sequence_id: The sequence ID of the entry to read. + * @param entry: Output param, the OpLog entry. + * @return: Error code. + */ + ErrorCode ReadOpLog(uint64_t sequence_id, OpLogEntry& entry); + + /** + * @brief Read OpLog entries starting from a given sequence_id. + * @param start_sequence_id: The starting sequence ID (exclusive). + * @param limit: Maximum number of entries to read (default: 1000). + * @param entries: Output param, vector of OpLog entries. + * @return: Error code. + */ + ErrorCode ReadOpLogSince(uint64_t start_sequence_id, size_t limit, + std::vector& entries); + + // Like ReadOpLogSince, but also returns the etcd revision for consistent + // "read then watch(from revision+1)" startup. + ErrorCode ReadOpLogSinceWithRevision(uint64_t start_sequence_id, size_t limit, + std::vector& entries, + EtcdRevisionId& revision_id); + + /** + * @brief Get the latest sequence_id from etcd. + * @param sequence_id: Output param, the latest sequence_id. + * @return: Error code. ETCD_KEY_NOT_EXIST if no OpLog exists yet. + */ + ErrorCode GetLatestSequenceId(uint64_t& sequence_id); + + // Stronger (than `/latest`) best-effort query: return the maximum existing + // sequence_id by scanning etcd keys under /oplog/{cluster_id}/ with + // descending key order. + // Return ETCD_KEY_NOT_EXIST if no OpLog exists yet. + ErrorCode GetMaxSequenceId(uint64_t& sequence_id); + + /** + * @brief Update the latest sequence_id in etcd. + * @param sequence_id: The latest sequence_id to update. + * @return: Error code. + */ + ErrorCode UpdateLatestSequenceId(uint64_t sequence_id); + + /** + * @brief Record the sequence_id corresponding to a snapshot. + * @param snapshot_id: The snapshot ID. + * @param sequence_id: The sequence_id at which the snapshot was taken. + * @return: Error code. + */ + ErrorCode RecordSnapshotSequenceId(const std::string& snapshot_id, + uint64_t sequence_id); + + /** + * @brief Get the sequence_id for a given snapshot. + * @param snapshot_id: The snapshot ID. + * @param sequence_id: Output param, the sequence_id. + * @return: Error code. ETCD_KEY_NOT_EXIST if snapshot not found. + */ + ErrorCode GetSnapshotSequenceId(const std::string& snapshot_id, + uint64_t& sequence_id); + + /** + * @brief Clean up OpLog entries before a given sequence_id. + * @param before_sequence_id: All entries with sequence_id < before_sequence_id + * will be deleted. + * @return: Error code. + */ + ErrorCode CleanupOpLogBefore(uint64_t before_sequence_id); + + /** + * @brief Destructor - stops batch update thread. + */ + ~EtcdOpLogStore(); + + private: + /** + * @brief Build the etcd key for an OpLog entry. + * @param sequence_id: The sequence ID. + * @return: The etcd key. + */ + std::string BuildOpLogKey(uint64_t sequence_id) const; + + /** + * @brief Build the etcd key for the latest sequence_id. + * @return: The etcd key. + */ + std::string BuildLatestKey() const; + + /** + * @brief Build the etcd key for a snapshot sequence_id. + * @param snapshot_id: The snapshot ID. + * @return: The etcd key. + */ + std::string BuildSnapshotKey(const std::string& snapshot_id) const; + + // Best-effort: find the minimum existing OpLog sequence_id in etcd. + // Used for robust cleanup (Scheme 3) so we don't rely on a persisted + // "cleaned_upto" marker. + std::optional GetMinSequenceId() const; + + // Best-effort: find the maximum existing OpLog sequence_id in etcd. + std::optional GetMaxSequenceIdInternal() const; + + /** + * @brief Serialize an OpLogEntry to JSON string. + * @param entry: The OpLog entry to serialize. + * @return: The JSON string. + */ + std::string SerializeOpLogEntry(const OpLogEntry& entry) const; + + /** + * @brief Deserialize a JSON string to OpLogEntry. + * @param json_str: The JSON string. + * @param entry: Output param, the OpLog entry. + * @return: true if successful, false otherwise. + */ + bool DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) const; + + /** + * @brief Batch update thread function. + * Periodically updates latest_sequence_id in etcd. + */ + void BatchUpdateThread(); + + /** + * @brief Trigger immediate batch update if threshold is reached. + */ + void TriggerBatchUpdateIfNeeded(); + + /** + * @brief Perform the actual batch update to etcd. + */ + void DoBatchUpdate(); + + std::string cluster_id_; + static constexpr const char* kOpLogPrefix = "/oplog/"; + static constexpr const char* kLatestSuffix = "/latest"; + static constexpr const char* kSnapshotPrefix = "/oplog/"; + static constexpr const char* kSnapshotSuffix = "/snapshot/"; + + // Batch update mechanism for latest_sequence_id + const bool enable_latest_seq_batch_update_{false}; + std::atomic pending_latest_seq_id_{0}; + std::atomic pending_count_{0}; + std::atomic batch_update_running_{false}; + std::mutex batch_update_mutex_; + std::thread batch_update_thread_; + std::chrono::steady_clock::time_point last_update_time_; + + // Batch update configuration + static constexpr size_t kBatchSize = 100; // Update every 100 entries + static constexpr int kBatchIntervalMs = 1000; // Or every 1 second +}; + +} // namespace mooncake diff --git a/mooncake-store/include/ha_helper.h b/mooncake-store/include/ha_helper.h index 897ba53a5c..a117cb29ef 100644 --- a/mooncake-store/include/ha_helper.h +++ b/mooncake-store/include/ha_helper.h @@ -3,12 +3,15 @@ #include +#include +#include #include #include #include -#include "types.h" +#include "hot_standby_service.h" #include "master_config.h" +#include "types.h" namespace mooncake { @@ -23,7 +26,18 @@ class MasterViewHelper { public: MasterViewHelper(const MasterViewHelper&) = delete; MasterViewHelper& operator=(const MasterViewHelper&) = delete; - MasterViewHelper(); + // cluster_id source of truth: + // - If provided, use it. + // - Else fall back to env MC_STORE_CLUSTER_ID. + // - Else fall back to DEFAULT_CLUSTER_ID. + explicit MasterViewHelper(const std::string& cluster_id = std::string()); + + // Update cluster_id (and derived master_view_key_) before using the helper. + // This is mainly for client-side etcd:// usage where cluster_id may be passed + // via connection string. + void SetClusterId(const std::string& cluster_id); + + const std::string& GetMasterViewKey() const { return master_view_key_; } /* * @brief Connect to the etcd cluster. This function should be called at @@ -60,6 +74,7 @@ class MasterViewHelper { ViewVersionId& version); private: + void BuildMasterViewKeyFromClusterId(const std::string& cluster_id); std::string master_view_key_; }; @@ -77,10 +92,27 @@ class MasterServiceSupervisor { ~MasterServiceSupervisor(); private: + /** + * @brief Start HotStandbyService when there is an existing leader + * @param mv_helper MasterViewHelper instance + * @param current_leader Current leader address + */ + void StartStandbyService(MasterViewHelper& mv_helper, + const std::string& current_leader); + + /** + * @brief Stop HotStandbyService + */ + void StopStandbyService(); + // coro_rpc server thread std::thread server_thread_; MasterServiceSupervisorConfig config_; + + // HotStandbyService for standby mode + std::unique_ptr standby_service_; + std::atomic standby_running_{false}; }; } // namespace mooncake diff --git a/mooncake-store/include/ha_metric_manager.h b/mooncake-store/include/ha_metric_manager.h new file mode 100644 index 0000000000..feebcc59ed --- /dev/null +++ b/mooncake-store/include/ha_metric_manager.h @@ -0,0 +1,192 @@ +#pragma once + +#include +#include +#include +#include + +#include "ylt/metric/counter.hpp" +#include "ylt/metric/gauge.hpp" +#include "ylt/metric/histogram.hpp" + +namespace mooncake { + +/** + * @brief Singleton manager for High Availability (HA) related metrics. + * + * This class provides metrics for monitoring the health and performance + * of the OpLog replication system, including: + * - OpLog sequence tracking + * - Standby replication lag + * - Error counters (checksum failures, skipped entries) + * - Performance histograms (etcd write latency) + * - Queue sizes (pending mutations) + */ +class HAMetricManager { + public: + // --- Singleton Access --- + static HAMetricManager& instance(); + + HAMetricManager(const HAMetricManager&) = delete; + HAMetricManager& operator=(const HAMetricManager&) = delete; + HAMetricManager(HAMetricManager&&) = delete; + HAMetricManager& operator=(HAMetricManager&&) = delete; + + // ========== OpLog Sequence Metrics (Gauge) ========== + + /** + * @brief Set the latest OpLog sequence ID on Primary + */ + void set_oplog_last_sequence_id(int64_t seq_id); + int64_t get_oplog_last_sequence_id(); + + /** + * @brief Set the Standby's applied sequence ID + */ + void set_oplog_applied_sequence_id(int64_t seq_id); + int64_t get_oplog_applied_sequence_id(); + + /** + * @brief Set the replication lag (entries behind Primary) + */ + void set_oplog_standby_lag(int64_t lag); + int64_t get_oplog_standby_lag(); + + /** + * @brief Set the number of pending (out-of-order) entries in OpLogApplier + */ + void set_oplog_pending_entries(int64_t count); + int64_t get_oplog_pending_entries(); + + /** + * @brief Set the pending mutation queue size (retry queue) + */ + void set_pending_mutation_queue_size(int64_t size); + int64_t get_pending_mutation_queue_size(); + + // ========== Error Counters ========== + + /** + * @brief Increment counter for skipped OpLog entries + */ + void inc_oplog_skipped_entries(int64_t val = 1); + int64_t get_oplog_skipped_entries_total(); + + /** + * @brief Increment counter for checksum verification failures + */ + void inc_oplog_checksum_failures(int64_t val = 1); + int64_t get_oplog_checksum_failures_total(); + + /** + * @brief Increment counter for gap resolve attempts + */ + void inc_oplog_gap_resolve_attempts(int64_t val = 1); + int64_t get_oplog_gap_resolve_attempts_total(); + + /** + * @brief Increment counter for successful gap resolves + */ + void inc_oplog_gap_resolve_success(int64_t val = 1); + int64_t get_oplog_gap_resolve_success_total(); + + /** + * @brief Increment counter for etcd write failures + */ + void inc_oplog_etcd_write_failures(int64_t val = 1); + int64_t get_oplog_etcd_write_failures_total(); + + /** + * @brief Increment counter for etcd write retries + */ + void inc_oplog_etcd_write_retries(int64_t val = 1); + int64_t get_oplog_etcd_write_retries_total(); + + /** + * @brief Increment counter for watch disconnections + */ + void inc_oplog_watch_disconnections(int64_t val = 1); + int64_t get_oplog_watch_disconnections_total(); + + /** + * @brief Increment counter for successfully applied OpLog entries + */ + void inc_oplog_applied_entries(int64_t val = 1); + int64_t get_oplog_applied_entries_total(); + + // ========== Latency Histograms ========== + + /** + * @brief Record etcd write latency in microseconds + */ + void observe_oplog_etcd_write_latency_us(int64_t latency_us); + + /** + * @brief Record OpLog apply latency in microseconds + */ + void observe_oplog_apply_latency_us(int64_t latency_us); + + // ========== State Machine Metrics ========== + + /** + * @brief Set the current Standby state (as integer for Prometheus) + * @param state_value Integer representation of StandbyState + */ + void set_standby_state(int64_t state_value); + int64_t get_standby_state(); + + /** + * @brief Increment state transition counter + */ + void inc_state_transitions(int64_t val = 1); + int64_t get_state_transitions_total(); + + // ========== Serialization ========== + + /** + * @brief Serializes all HA metrics into Prometheus text format. + * @return A string containing the metrics in Prometheus format. + */ + std::string serialize_metrics(); + + /** + * @brief Generates a concise, human-readable summary of HA metrics. + * @return A string containing the formatted summary. + */ + std::string get_summary_string(); + + private: + // --- Private Constructor & Destructor --- + HAMetricManager(); + ~HAMetricManager() = default; + + // --- Metric Members --- + + // OpLog Sequence Gauges + ylt::metric::gauge_t oplog_last_sequence_id_; + ylt::metric::gauge_t oplog_applied_sequence_id_; + ylt::metric::gauge_t oplog_standby_lag_; + ylt::metric::gauge_t oplog_pending_entries_; + ylt::metric::gauge_t pending_mutation_queue_size_; + + // Error Counters + ylt::metric::counter_t oplog_skipped_entries_total_; + ylt::metric::counter_t oplog_checksum_failures_total_; + ylt::metric::counter_t oplog_gap_resolve_attempts_total_; + ylt::metric::counter_t oplog_gap_resolve_success_total_; + ylt::metric::counter_t oplog_etcd_write_failures_total_; + ylt::metric::counter_t oplog_etcd_write_retries_total_; + ylt::metric::counter_t oplog_watch_disconnections_total_; + ylt::metric::counter_t oplog_applied_entries_total_; + + // Latency Histograms (buckets in microseconds: 100us, 500us, 1ms, 5ms, 10ms, 50ms, 100ms, 500ms, 1s) + ylt::metric::histogram_t oplog_etcd_write_latency_us_; + ylt::metric::histogram_t oplog_apply_latency_us_; + + // State Machine + ylt::metric::gauge_t standby_state_; + ylt::metric::counter_t state_transitions_total_; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/hot_standby_service.h b/mooncake-store/include/hot_standby_service.h new file mode 100644 index 0000000000..29544775ed --- /dev/null +++ b/mooncake-store/include/hot_standby_service.h @@ -0,0 +1,244 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "metadata_store.h" +#include "oplog_applier.h" +#include "oplog_manager.h" +#include "oplog_watcher.h" +#include "snapshot_provider.h" +#include "standby_state_machine.h" +#include "types.h" + +namespace mooncake { + +// Forward declarations +class MasterService; +class ReplicationStream; + +/** + * @brief Configuration for HotStandbyService + */ +struct HotStandbyConfig { + std::string standby_id; + std::string primary_address; + uint32_t replication_port{0}; + uint32_t verification_interval_sec{30}; + uint32_t max_replication_lag_entries{1000}; + bool enable_verification{true}; + + // Snapshot bootstrap (optional): + // If provided, Standby will try to load a snapshot first, then replay OpLog + // from snapshot_sequence_id. + bool enable_snapshot_bootstrap{false}; +}; + +/** + * @brief Sync status information for HotStandbyService + */ +struct StandbySyncStatus { + uint64_t applied_seq_id{0}; + uint64_t primary_seq_id{0}; + uint64_t lag_entries{0}; + std::chrono::milliseconds lag_time{0}; + bool is_syncing{false}; + bool is_connected{false}; + StandbyState state{StandbyState::STOPPED}; + std::chrono::milliseconds time_in_state{0}; +}; + +/** + * @brief HotStandbyService manages standby replication and promotion + * + * This service runs on Standby Master nodes and is responsible for: + * - Connecting to Primary and receiving OpLog entries + * - Applying OpLog entries to local metadata store + * - Periodically verifying data consistency with Primary + * - Promoting to Primary when elected as new Leader + * + * For now, this is a skeleton implementation without actual network + * communication. The gRPC integration will be added later. + */ +class HotStandbyService { + public: + explicit HotStandbyService(const HotStandbyConfig& config); + ~HotStandbyService(); + + /** + * @brief Start connecting to Primary and begin replication + * @param primary_address Address of the Primary Master (not used with etcd-based sync) + * @param etcd_endpoints Comma-separated etcd endpoints + * @param cluster_id Cluster identifier for OpLog path + * @return ErrorCode::OK on success + */ + ErrorCode Start(const std::string& primary_address, + const std::string& etcd_endpoints, + const std::string& cluster_id); + + /** + * @brief Stop replication and disconnect from Primary + */ + void Stop(); + + /** + * @brief Get current synchronization status + * @return StandbySyncStatus with current sync state + */ + StandbySyncStatus GetSyncStatus() const; + + /** + * @brief Check if standby is ready for promotion + * @return true if replication lag is within threshold + */ + bool IsReadyForPromotion() const; + + /** + * @brief Promote this standby to Primary + * + * This method should be called after successful leader election. + * It returns a MasterService instance initialized with the replicated + * metadata, ready to serve as the new Primary. + * + * @return Unique pointer to MasterService, or nullptr on failure + */ + std::unique_ptr Promote(); + + /** + * @brief Get the number of metadata entries in the local store + */ + size_t GetMetadataCount() const; + + /** + * @brief Get the latest applied sequence ID after promotion + * + * This should be called after Promote() to get the sequence_id + * that the new Primary's OpLogManager should start from. + * + * @return Latest applied sequence ID, or 0 if not available + */ + uint64_t GetLatestAppliedSequenceId() const; + + // Export a point-in-time snapshot of all replicated metadata. + // This is used by MasterServiceSupervisor to initialize the new Primary + // after leader election (fast recovery). + bool ExportMetadataSnapshot( + std::vector>& out) const; + + // Inject a snapshot provider (from external snapshot implementation). + void SetSnapshotProvider(std::unique_ptr provider); + + /** + * @brief Get current state from state machine + */ + StandbyState GetState() const { return state_machine_.GetState(); } + + /** + * @brief Get state machine for monitoring/debugging + */ + const StandbyStateMachine& GetStateMachine() const { return state_machine_; } + + /** + * @brief Callback for OpLogWatcher state changes + * @param event The event to process + */ + void OnWatcherEvent(StandbyEvent event); + + private: + /** + * @brief Main replication loop (runs in background thread) + */ + void ReplicationLoop(); + + /** + * @brief Verification loop (runs in background thread) + */ + void VerificationLoop(); + + /** + * @brief Apply a single OpLog entry to local metadata store + * @param entry The OpLog entry to apply + * @deprecated Use OpLogApplier instead + */ + void ApplyOpLogEntry(const OpLogEntry& entry); + + /** + * @brief Connect to Primary and establish replication stream + * @return true on success, false on failure + */ + bool ConnectToPrimary(); + + /** + * @brief Disconnect from Primary + */ + void DisconnectFromPrimary(); + + /** + * @brief Process a batch of OpLog entries received from Primary + * @param entries Batch of OpLog entries + */ + void ProcessOpLogBatch(const std::vector& entries); + + HotStandbyConfig config_; + + // Simple in-memory metadata store implementation + class StandbyMetadataStore : public MetadataStore { + public: + bool PutMetadata(const std::string& key, + const StandbyObjectMetadata& metadata) override; + bool Put(const std::string& key, + const std::string& payload = std::string()) override; + const StandbyObjectMetadata* GetMetadata(const std::string& key) const override; + bool Remove(const std::string& key) override; + bool Exists(const std::string& key) const override; + size_t GetKeyCount() const override; + + // Snapshot for promotion/restore. + void Snapshot( + std::vector>& out) const; + + private: + mutable std::mutex mutex_; + std::unordered_map store_; + }; + std::unique_ptr metadata_store_; + std::unique_ptr snapshot_provider_{std::make_unique()}; + + // OpLog replication components + std::unique_ptr oplog_applier_; + std::unique_ptr oplog_watcher_; + + // Configuration for etcd-based OpLog sync + std::string etcd_endpoints_; + std::string cluster_id_; + + // Replication state + std::shared_ptr replication_stream_; + std::atomic applied_seq_id_{0}; + std::atomic primary_seq_id_{0}; + + // State machine for managing service lifecycle + StandbyStateMachine state_machine_; + + // Helper methods for state machine + bool IsRunning() const { return state_machine_.IsRunning(); } + bool IsConnected() const { return state_machine_.IsConnected(); } + + // Background threads + std::thread replication_thread_; + std::thread verification_thread_; + + // Synchronization + mutable std::mutex mutex_; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 35e728f0c5..574d050f48 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -4,7 +4,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -13,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -25,11 +29,16 @@ #include "master_config.h" #include "rpc_types.h" #include "replica.h" +#include "oplog_manager.h" +#include "metadata_store.h" namespace mooncake { // Forward declarations class AllocationStrategy; class EvictionStrategy; +class BufferAllocatorBase; +struct StandbyObjectMetadata; +// ReplicationService forward declaration removed - using etcd-based OpLog sync instead /* * @brief MasterService is the main class for the master server. @@ -94,6 +103,13 @@ class MasterService { */ auto GetAllKeys() -> tl::expected, ErrorCode>; + // Restore metadata from a Standby snapshot (fast failover). + // NOTE: This is used only on the node that was running HotStandbyService + // right before it was promoted to leader. + void RestoreFromStandbySnapshot( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id); + /** * @brief Fetch all segments, each node has a unique real client with fixed * segment name : segment name, preferred format : {ip}:{port}, bad format : @@ -271,6 +287,14 @@ class MasterService { */ tl::expected GetStorageConfig() const; + /** + * @brief Get OpLogManager reference for external access + * @return Reference to the OpLogManager instance + */ + OpLogManager& GetOpLogManager(); + + // SetReplicationService removed - using etcd-based OpLog sync instead + /** * @brief Mounts a file storage segment into the master. * @param enable_offloading If true, enables offloading (write-to-file). @@ -301,6 +325,21 @@ class MasterService { -> tl::expected; private: + /** + * @brief Helper function to append OpLog entry + * @param type Operation type + * @param key Object key + * @param payload Optional payload data + */ + void AppendOpLogAndNotify(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Durable OpLog append: must succeed (write to etcd) for operations that may + // free/reuse memory (e.g. REMOVE). See OpLogManager::AppendAndPersist. + auto AppendOpLogAndNotifyDurable(OpType type, const std::string& key, + const std::string& payload = std::string()) + -> tl::expected; + // Resolve the key to a sanitized format for storage std::string SanitizeKey(const std::string& key) const; std::string ResolvePath(const std::string& key) const; @@ -477,6 +516,95 @@ class MasterService { } }; + /** + * @brief Serialize ObjectMetadata to JSON string for OpLog payload + * @param metadata The metadata to serialize + * @return JSON string containing the serialized metadata + */ + std::string SerializeMetadataForOpLog(const ObjectMetadata& metadata) const; + + // Serialize metadata but exclude MEMORY replicas. + // Used for eviction: when memory replicas are freed/reused, Standby must not + // keep stale memory descriptors. We persist a PUT_END containing only + // remaining (DISK/LOCAL_DISK) replicas before freeing memory. + std::string SerializeMetadataForOpLogWithoutMemReplicas( + const ObjectMetadata& metadata) const; + + // Serialize metadata from a caller-provided replica descriptor list. + // This is used when we need to persist an updated replica set *before* + // mutating local replicas (which may free/reuse memory). + std::string SerializeMetadataForOpLogFromReplicaDescriptors( + const UUID& client_id, uint64_t size, + const std::vector& replicas) const; + + // Pending durable mutations (etcd write retry queue) + // -------------------------------------------------- + // In HA mode, freeing/reusing MEMORY replicas before Standby observes the + // corresponding OpLog update can cause stale descriptors on Standby. + // If durable etcd write fails, we enqueue a pending mutation and retry + // asynchronously to avoid long-term memory retention. + enum class PendingMutationKind : uint8_t { + EVICT_MEM_REPLICAS = 1, // drop MEMORY replicas; persist PUT_END or REMOVE + CLEAR_ALL_REPLICAS = 2, // remove the whole key; persist REMOVE + CLEAR_REPLICAS_ON_SEGMENT = 3, // remove COMPLETE replicas on segment; persist PUT_END/REMOVE + }; + struct PendingMutation { + PendingMutationKind kind{PendingMutationKind::EVICT_MEM_REPLICAS}; + std::string key; + std::string segment_name; // only for CLEAR_REPLICAS_ON_SEGMENT + // OpLog entry to persist. If sequence_id==0, this is a deferred action and + // the worker will allocate a new OpLogEntry at execution time. + // If sequence_id>0, sequence_id is pre-allocated and MUST be persisted as-is + // (implements: "enqueue time seq_id fixed and smaller"). + OpLogEntry oplog_entry; + uint32_t attempt{0}; + std::chrono::steady_clock::time_point next_retry_at{}; + }; + + void EnqueuePendingMutation(PendingMutation m); + void PendingMutationWorker(); + bool ProcessPendingMutationOnce(PendingMutation& m); + + // Helper for etcd durable write (HA only): + // - Persist a pre-allocated OpLogEntry with small synchronous retries. + // - On failure, enqueue a PendingMutation (caller decides whether to proceed + // with local state changes; we do NOT block per-key). + ErrorCode PersistOpLogEntryWithSyncRetries(const OpLogEntry& entry) const; + void EnqueueRetryOnPersistFailure(const char* ctx, const OpLogEntry& entry, + ErrorCode persist_err, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // Higher-level helper that also handles: + // - STORE_USE_ETCD compile-time switch + // - enable_ha_ runtime switch + // + // Behavior: + // - HA + STORE_USE_ETCD: AllocateEntry -> Persist (sync retries) -> enqueue on failure + // - non-HA: Append to in-memory OpLog buffer only + // - HA but STORE_USE_ETCD disabled: no-op (best-effort; see constructor warning) + void AppendOrPersistOrEnqueue(const char* ctx, OpType type, + const std::string& key, + const std::string& payload, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // Lazy-payload variant: payload is computed only when needed. + // This is useful to avoid expensive metadata serialization when: + // - HA is enabled but STORE_USE_ETCD is disabled at compile time (no-op), or + // - the branch will not publish OpLog at all. + void AppendOrPersistOrEnqueueLazy( + const char* ctx, OpType type, const std::string& key, + const std::function& payload_factory, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // NOTE: + // We intentionally do NOT block subsequent operations for the same key when a + // durable OpLog write fails. Failed entries are retried asynchronously with the + // original pre-allocated sequence_id, and Standby handles gaps via timeout + + // late-arrival policy (apply late REMOVE/PUT_REVOKE, discard late PUT_END). + static constexpr size_t kNumShards = 1024; // Number of metadata shards // Sharded metadata maps and their mutexes @@ -631,8 +759,30 @@ class MasterService { // Segment management SegmentManager segment_manager_; BufferAllocatorType memory_allocator_type_; + + // Keep dummy allocators alive for memory replicas restored from standby. + // AllocatedBuffer stores allocator as weak_ptr; without an owning shared_ptr, + // the allocator would expire immediately and transport_endpoint_ would be lost + // when re-serializing Replica descriptors. + std::unordered_map> + standby_allocator_keepalive_; + + // Operation log manager for hot-standby replication. It records + // state-changing operations so that a standby master can replay them. + OpLogManager oplog_manager_; + + // ReplicationService removed - using etcd-based OpLog sync instead + std::shared_ptr allocation_strategy_; + // Pending durable mutation retry queue (HA only). + std::mutex pending_mutations_mutex_; + std::condition_variable pending_mutations_cv_; + std::deque pending_mutations_; + std::atomic pending_mutations_running_{false}; + std::thread pending_mutations_thread_; + static constexpr size_t kMaxPendingMutations = 10000; // Max queue size to prevent unbounded growth + // Discarded replicas management const std::chrono::seconds put_start_discard_timeout_sec_; const std::chrono::seconds put_start_release_timeout_sec_; diff --git a/mooncake-store/include/metadata_store.h b/mooncake-store/include/metadata_store.h new file mode 100644 index 0000000000..33842238f4 --- /dev/null +++ b/mooncake-store/include/metadata_store.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include +#include +#include + +#include "replica.h" +#include "types.h" +#include "ylt/struct_json/json_reader.h" +#include "ylt/struct_json/json_writer.h" + +namespace mooncake { + +/** + * @brief Metadata structure for Standby to store and restore object information + * + * This structure contains all essential metadata information needed by Standby + * to immediately serve as Primary when promoted. + */ +struct StandbyObjectMetadata { + UUID client_id{0, 0}; + uint64_t size{0}; + std::vector replicas; + // NOTE: Lease information is NOT stored because: + // 1. Standby does not perform eviction, so lease info is not used + // 2. After promotion, new Primary should grant fresh leases, not restore old ones + uint64_t last_sequence_id{0}; // Last OpLog sequence ID that modified this key + + StandbyObjectMetadata() = default; + + // Check if this metadata has valid replicas + bool HasReplicas() const { return !replicas.empty(); } +}; + +/** + * @brief Payload structure for JSON serialization/deserialization + * + * Uses separate fields for UUID since std::pair cannot be directly serialized. + */ +struct MetadataPayload { + uint64_t client_id_first{0}; // UUID.first + uint64_t client_id_second{0}; // UUID.second + uint64_t size{0}; + std::vector replicas; + // NOTE: Lease information removed - not needed by Standby + + YLT_REFL(MetadataPayload, client_id_first, client_id_second, size, replicas); + + // Convert to StandbyObjectMetadata + StandbyObjectMetadata ToStandbyMetadata(uint64_t sequence_id) const { + StandbyObjectMetadata meta; + meta.client_id = {client_id_first, client_id_second}; + meta.size = size; + meta.replicas = replicas; + meta.last_sequence_id = sequence_id; + return meta; + } +}; + +/** + * @brief Abstract interface for metadata storage on Standby + * + * This interface provides basic operations for storing and managing object metadata. + * In a full implementation, this would mirror MasterService's metadata_shards_ structure. + */ +class MetadataStore { + public: + virtual ~MetadataStore() = default; + + /** + * @brief Put or update metadata for a key with structured metadata + * @param key Object key + * @param metadata Structured metadata object + * @return true on success, false on failure + */ + virtual bool PutMetadata(const std::string& key, const StandbyObjectMetadata& metadata) = 0; + + /** + * @brief Put or update metadata for a key (legacy interface for backward compatibility) + * @param key Object key + * @param payload Optional payload data (JSON serialized metadata) + * @return true on success, false on failure + */ + virtual bool Put(const std::string& key, const std::string& payload = std::string()) = 0; + + /** + * @brief Get metadata for a key + * @param key Object key + * @return Pointer to metadata if found, nullptr otherwise + */ + virtual const StandbyObjectMetadata* GetMetadata(const std::string& key) const = 0; + + /** + * @brief Remove metadata for a key + * @param key Object key + * @return true if key was found and removed, false otherwise + */ + virtual bool Remove(const std::string& key) = 0; + + /** + * @brief Check if a key exists + * @param key Object key + * @return true if key exists, false otherwise + */ + virtual bool Exists(const std::string& key) const = 0; + + /** + * @brief Get the count of keys in the store + * @return Number of keys + */ + virtual size_t GetKeyCount() const = 0; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/oplog_applier.h b/mooncake-store/include/oplog_applier.h new file mode 100644 index 0000000000..993ef81f5c --- /dev/null +++ b/mooncake-store/include/oplog_applier.h @@ -0,0 +1,170 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "metadata_store.h" + +namespace mooncake { + +// Forward declaration +class EtcdOpLogStore; + +/** + * @brief Apply OpLog entries to Standby metadata store with ordering guarantee + * + * This class applies OpLog entries to the Standby metadata store, + * ensuring both global and per-key ordering. + */ +class OpLogApplier { + public: + /** + * @brief Constructor + * @param metadata_store Metadata store to apply changes to + * @param cluster_id Cluster ID for accessing etcd OpLog (optional, for requesting missing OpLog) + */ + explicit OpLogApplier(MetadataStore* metadata_store, + const std::string& cluster_id = std::string()); + + /** + * @brief Apply a single OpLog entry (with ordering checks) + * @param entry OpLog entry to apply + * @return true on success, false on failure or ordering violation + */ + bool ApplyOpLogEntry(const OpLogEntry& entry); + + /** + * @brief Apply multiple OpLog entries + * @param entries OpLog entries to apply + * @return Number of successfully applied entries + */ + size_t ApplyOpLogEntries(const std::vector& entries); + + /** + * @brief Get the current sequence ID for a key (DEPRECATED) + * @param key Object key + * @return Always returns 0 - global sequence_id is used for ordering + * @deprecated Use global sequence_id for ordering + */ + uint64_t GetKeySequenceId(const std::string& key) const; + + /** + * @brief Get the expected global sequence ID + * @return Expected global sequence ID + */ + uint64_t GetExpectedSequenceId() const; + + /** + * @brief Recover from a given sequence ID + * @param last_applied_sequence_id Last applied sequence ID + */ + void Recover(uint64_t last_applied_sequence_id); + + /** + * @brief Process pending entries (entries with non-continuous sequence IDs) + * @return Number of entries processed + */ + size_t ProcessPendingEntries(); + + // Promotion helper: + // Try to resolve current gaps ONCE (no waiting) by fetching missing/skipped + // sequence_ids from etcd. If an entry arrives late: + // - REMOVE / PUT_REVOKE: delete the key + // - PUT_END: discard + // + // This is used during Standby promotion so we don't block promotion on gaps, + // but still best-effort clean up potentially stale metadata. + struct GapResolveResult { + size_t attempted{0}; + size_t fetched{0}; + size_t applied_deletes{0}; + }; + GapResolveResult TryResolveGapsOnceForPromotion(size_t max_ids = 1024); + + private: + /** + * @brief Check if the entry's sequence order is valid + * @param entry OpLog entry + * @return true if order is valid, false otherwise + */ + bool CheckSequenceOrder(const OpLogEntry& entry); + + /** + * @brief Apply PUT_END operation + * @param entry OpLog entry + */ + void ApplyPutEnd(const OpLogEntry& entry); + + /** + * @brief Apply PUT_REVOKE operation + * @param entry OpLog entry + */ + void ApplyPutRevoke(const OpLogEntry& entry); + + /** + * @brief Apply REMOVE operation + * @param entry OpLog entry + */ + void ApplyRemove(const OpLogEntry& entry); + + /** + * @brief Request missing OpLog entry from etcd + * @param missing_seq_id Missing sequence ID + * @return true if entry was found and applied, false otherwise + */ + bool RequestMissingOpLog(uint64_t missing_seq_id); + + /** + * @brief Schedule wait for missing entries + * @param missing_seq_id Missing sequence ID + */ + void ScheduleWaitForMissingEntries(uint64_t missing_seq_id); + + MetadataStore* metadata_store_; + + // EtcdOpLogStore for requesting missing OpLog entries (optional) + std::string cluster_id_; + mutable std::mutex etcd_oplog_store_mutex_; + mutable std::unique_ptr etcd_oplog_store_; + + /** + * @brief Get or create EtcdOpLogStore instance (lazy initialization) + * @return Pointer to EtcdOpLogStore, or nullptr if cluster_id is not set + */ + EtcdOpLogStore* GetEtcdOpLogStore() const; + + // Note: key_sequence_map_ has been removed. + // Global sequence_id is sufficient for ordering guarantee. + + // Track pending entries (entries with non-continuous sequence IDs) + mutable std::mutex pending_mutex_; + std::map pending_entries_; + + // Track missing sequence IDs that we're waiting for + std::map missing_sequence_ids_; + + // Sequence IDs we chose to skip (gap-timeout). If the late entry arrives: + // - REMOVE / PUT_REVOKE: delete the key (safe) + // - PUT_END: discard (do not resurrect potentially stale metadata) + std::map skipped_sequence_ids_; + + // Next expected global sequence_id. Read frequently from monitoring thread, + // updated by watch/apply thread. Use atomic to avoid data races. + std::atomic expected_sequence_id_{1}; + + // Constants for missing entry handling + // IMPORTANT: request must happen BEFORE skip, otherwise we will never request. + static constexpr int kMissingEntryRequestSeconds = 1; // request from etcd after 1s + static constexpr int kMissingEntrySkipSeconds = 3; // skip after 3s (avoid global stall) + static constexpr int kMaxPendingEntries = 1000; // Max pending entries before giving up +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h new file mode 100644 index 0000000000..df76ce6eeb --- /dev/null +++ b/mooncake-store/include/oplog_manager.h @@ -0,0 +1,139 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "types.h" + +namespace mooncake { + +// Forward declaration +class EtcdOpLogStore; + +// Operation types for hot-standby replication. +// This is a minimal subset that can be extended later. +enum class OpType : uint8_t { + PUT_END = 1, + PUT_REVOKE = 2, + REMOVE = 3, + // Deprecated: LEASE_RENEW is intentionally not recorded in OpLog in the + // current etcd-based hot-standby design (Standby relies on Primary DELETE operations). + LEASE_RENEW = 4, +}; + +// A single operation log entry. +// Note: Payload contains JSON serialized MetadataPayload (defined in metadata_store.h) +// for PUT_END operations, allowing Standby to restore complete metadata. +struct OpLogEntry { + uint64_t sequence_id{0}; // Monotonically increasing global sequence + uint64_t timestamp_ms{0}; // Logical timestamp in milliseconds + OpType op_type{OpType::PUT_END}; + std::string object_key; // Target object key + std::string payload; // Serialized extra data (optional) + uint32_t checksum{0}; // Checksum of payload (implementation-defined) + uint32_t prefix_hash{0}; // Hash of the entire key (for verification and optimization) +}; + +/** + * @brief In-memory operation log manager. + * + * This class is intentionally simple: it keeps a bounded deque of OpLogEntry + * and provides append / get-since primitives. It can later be extended to + * or to spill to disk if needed. In the new etcd-based design, OpLog will be written to etcd. + */ +class OpLogManager { + public: + OpLogManager(); + + // Set the EtcdOpLogStore for writing OpLog to etcd (optional). + // If not set, OpLog will only be stored in memory buffer. + void SetEtcdOpLogStore(std::shared_ptr etcd_oplog_store); + + // Append a new entry and return the assigned sequence_id. + uint64_t Append(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Allocate a new OpLogEntry with a reserved sequence_id, append it to the + // in-memory buffer, and return the full entry. + // + // IMPORTANT: This will advance last_seq_id_ even if the caller later fails + // to persist it to etcd. This supports "seq pre-allocation" semantics where + // retries use the same (smaller) sequence_id. + OpLogEntry AllocateEntry(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Persist an already-allocated entry to etcd using its sequence_id. + // Does NOT modify sequence counters. + ErrorCode PersistEntryToEtcd(const OpLogEntry& entry) const; + + // Append a new entry and durably persist it to etcd (if EtcdOpLogStore is set). + // + // This is intended for operations that may free/reuse memory (e.g. REMOVE), + // where best-effort replication is unsafe: Standby must observe the DELETE + // before promotion, otherwise it may return stale descriptors that point to + // reused memory and cause silent data corruption. + // + // Design (updated for seq pre-allocation): + // - sequence_id is allocated first and never reused. + // - If etcd write fails, caller may retry PersistEntryToEtcd with the same + // entry (sequence_id fixed and "smaller" than later entries). + tl::expected AppendAndPersist( + OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Get the latest assigned sequence id. Returns 0 if no entry exists. + uint64_t GetLastSequenceId() const; + + // Set the initial sequence ID (used when promoting Standby to Primary). + // This ensures the new Primary's OpLogManager continues from the correct sequence_id. + void SetInitialSequenceId(uint64_t sequence_id); + + // Current number of entries in the buffer. + size_t GetEntryCount() const; + + // Verify checksum of an OpLogEntry payload. + // Returns true if checksum matches, false otherwise. + // This is public so OpLogWatcher and OpLogApplier can validate entries. + static bool VerifyChecksum(const OpLogEntry& entry); + + // Basic DoS protection for externally sourced OpLog entries (etcd watch / reads). + // Enforce conservative bounds on key/payload sizes before parsing/applying. + static constexpr size_t kMaxObjectKeySize = 4096; // 4 KiB + static constexpr size_t kMaxPayloadSize = 10 * 1024 * 1024; // 10 MiB + + // Validate OpLogEntry key/payload sizes. If invalid, returns false and + // optionally sets a human-readable reason. + static bool ValidateEntrySize(const OpLogEntry& entry, + std::string* reason = nullptr); + + private: + static uint64_t NowMs(); + static uint32_t ComputeChecksum(const std::string& data); + static uint32_t ComputePrefixHash(const std::string& key); + + mutable std::shared_mutex mutex_; + std::deque buffer_; + uint64_t first_seq_id_{1}; // sequence_id of buffer_.front() + uint64_t last_seq_id_{0}; // last assigned sequence_id + + // Note: We removed key_sequence_map_ and key_remove_time_map_. + // Global sequence_id is sufficient for ordering guarantee. + // All operations are applied in sequence_id order, which ensures consistency. + + // Optional etcd OpLog store for persistent storage + std::shared_ptr etcd_oplog_store_; + + // Simple bounds to avoid unbounded memory growth. + static constexpr size_t kMaxBufferEntries_ = 100000; +}; + +} // namespace mooncake + + diff --git a/mooncake-store/include/oplog_watcher.h b/mooncake-store/include/oplog_watcher.h new file mode 100644 index 0000000000..9c430e87be --- /dev/null +++ b/mooncake-store/include/oplog_watcher.h @@ -0,0 +1,158 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "standby_state_machine.h" +#include "types.h" + +namespace mooncake { + +// Forward declaration +class OpLogApplier; + +// Callback type for state events +using WatcherStateCallback = std::function; + +/** + * @brief Watch etcd for OpLog changes and apply them to Standby + * + * This class watches etcd for new OpLog entries and forwards them + * to OpLogApplier for processing. + */ +class OpLogWatcher { + public: + /** + * @brief Constructor + * @param etcd_endpoints Comma-separated etcd endpoints + * @param cluster_id Cluster identifier + * @param applier OpLog applier to process entries + */ + OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier); + + ~OpLogWatcher(); + + /** + * @brief Start watching etcd for OpLog changes + */ + void Start(); + + /** + * @brief Start from a known last-applied sequence_id. + * + * It will read historical OpLogs at a consistent etcd revision, then start + * watch from revision+1 to close the gap between "read" and "watch". + */ + bool StartFromSequenceId(uint64_t start_seq_id); + + /** + * @brief Stop watching + */ + void Stop(); + + /** + * @brief Get the last processed sequence ID + * @return Last processed sequence ID + */ + uint64_t GetLastProcessedSequenceId() const; + + /** + * @brief Set callback for state events + * @param callback Callback function to invoke on state events + */ + void SetStateCallback(WatcherStateCallback callback) { + state_callback_ = std::move(callback); + } + + /** + * @brief Check if watch is healthy + */ + bool IsWatchHealthy() const { return watch_healthy_.load(); } + + private: + /** + * @brief Notify state callback + */ + void NotifyStateEvent(StandbyEvent event) { + if (state_callback_) { + state_callback_(event); + } + } + + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries, + EtcdRevisionId& revision_id); + // Callback includes etcd KV mod_revision for precise resume. + static void WatchCallback(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, int event_type, + int64_t mod_revision); + + /** + * @brief Watch etcd OpLog changes (runs in background thread) + */ + void WatchOpLog(); + + /** + * @brief Process a Watch event + * @param key etcd key + * @param value etcd value (JSON string for PUT events, empty for DELETE events) + * @param event_type Event type (0 = PUT, 1 = DELETE) + */ + void HandleWatchEvent(const std::string& key, const std::string& value, + int event_type); + void HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision); + + /** + * @brief Deserialize OpLogEntry from JSON string + * @param json_str JSON string + * @param entry Output OpLog entry + * @return true on success, false on failure + */ + bool DeserializeOpLogEntry(const std::string& json_str, OpLogEntry& entry); + + /** + * @brief Attempt to reconnect after watch failure + */ + void TryReconnect(); + + /** + * @brief Sync missed OpLog entries after reconnection + * @return true if sync was successful + */ + bool SyncMissedEntries(); + + // Next watch revision (0 means from now). Updated by consistent reads. + std::atomic next_watch_revision_{0}; + + std::string etcd_endpoints_; + std::string cluster_id_; + OpLogApplier* applier_; + std::atomic running_{false}; + std::thread watch_thread_; + std::atomic last_processed_sequence_id_{0}; + + // Error handling and recovery + std::atomic consecutive_errors_{0}; + std::atomic reconnect_count_{0}; + std::atomic watch_healthy_{false}; + + // State callback for notifying HotStandbyService + WatcherStateCallback state_callback_; + + // Constants for error handling + static constexpr int kMaxConsecutiveErrors = 10; + static constexpr int kReconnectDelayMs = 1000; + static constexpr int kMaxReconnectDelayMs = 30000; + static constexpr int kSyncBatchSize = 1000; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/replica.h b/mooncake-store/include/replica.h index 5793e90f96..997259ca0b 100644 --- a/mooncake-store/include/replica.h +++ b/mooncake-store/include/replica.h @@ -131,10 +131,21 @@ struct DiskDescriptor { }; struct LocalDiskDescriptor { - UUID client_id; + uint64_t client_id_first{0}; // UUID.first - split for JSON serialization + uint64_t client_id_second{0}; // UUID.second - split for JSON serialization uint64_t object_size = 0; std::string transport_endpoint; - YLT_REFL(LocalDiskDescriptor, client_id, object_size, transport_endpoint); + + // Constructor from UUID for convenience + LocalDiskDescriptor() = default; + LocalDiskDescriptor(UUID client_id, uint64_t object_size, const std::string& transport_endpoint) + : client_id_first(client_id.first), client_id_second(client_id.second), + object_size(object_size), transport_endpoint(transport_endpoint) {} + + // Get UUID (for backward compatibility) + UUID GetClientId() const { return {client_id_first, client_id_second}; } + + YLT_REFL(LocalDiskDescriptor, client_id_first, client_id_second, object_size, transport_endpoint); }; class Replica { @@ -375,10 +386,9 @@ inline Replica::Descriptor Replica::get_descriptor() const { desc.descriptor_variant = std::move(disk_desc); } else if (is_local_disk_replica()) { const auto& disk_data = std::get(data_); - LocalDiskDescriptor local_disk_desc; - local_disk_desc.client_id = disk_data.client_id; - local_disk_desc.object_size = disk_data.object_size; - local_disk_desc.transport_endpoint = disk_data.transport_endpoint; + LocalDiskDescriptor local_disk_desc(disk_data.client_id, + disk_data.object_size, + disk_data.transport_endpoint); desc.descriptor_variant = std::move(local_disk_desc); } diff --git a/mooncake-store/include/rpc_service.h b/mooncake-store/include/rpc_service.h index 1cf5d7a24f..465a6d5661 100644 --- a/mooncake-store/include/rpc_service.h +++ b/mooncake-store/include/rpc_service.h @@ -4,15 +4,22 @@ #include #include #include +#include #include #include #include #include "master_service.h" +#include "metadata_store.h" #include "types.h" #include "rpc_types.h" #include "master_config.h" +// Forward declaration +namespace mooncake { +// ReplicationService forward declaration removed - using etcd-based OpLog sync instead +} + namespace mooncake { extern const uint64_t kMetricReportIntervalSeconds; @@ -25,6 +32,11 @@ class WrappedMasterService { void init_http_server(); + // Restore metadata and OpLog sequence from a promoted Standby (fast failover). + void RestoreFromStandby( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id); + tl::expected ExistKey(const std::string& key); tl::expected @@ -109,11 +121,15 @@ class WrappedMasterService { const UUID& client_id, const std::vector& keys, const std::vector& metadatas); + // GetReplicationService removed - using etcd-based OpLog sync instead + private: MasterService master_service_; std::thread metric_report_thread_; coro_http::coro_http_server http_server_; std::atomic metric_report_running_; + + // ReplicationService removed - using etcd-based OpLog sync instead }; void RegisterRpcService(coro_rpc::coro_rpc_server& server, diff --git a/mooncake-store/include/snapshot_provider.h b/mooncake-store/include/snapshot_provider.h new file mode 100644 index 0000000000..ae8041511c --- /dev/null +++ b/mooncake-store/include/snapshot_provider.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +#include "metadata_store.h" + +namespace mooncake { + +/** + * @brief SnapshotProvider is an abstraction for loading metadata snapshots. + * + * Assumption: snapshot functionality exists (implemented by another team), but + * may not be synced into this repo yet. We keep Mooncake-store code progressing + * by depending on this narrow interface. + * + * Snapshot semantics for hot-standby: + * - A snapshot represents a consistent metadata baseline at `snapshot_sequence_id`. + * - Standby should: load snapshot -> recover applier to snapshot_sequence_id -> + * replay OpLog entries with sequence_id > snapshot_sequence_id. + */ +class SnapshotProvider { + public: + virtual ~SnapshotProvider() = default; + + // Load the latest available snapshot for `cluster_id`. + // Returns true on success and fills: + // - snapshot_id: opaque identifier (e.g. timestamp/version) + // - snapshot_sequence_id: global OpLog sequence_id at snapshot boundary + // - snapshot: full metadata baseline as key -> StandbyObjectMetadata + virtual bool LoadLatestSnapshot( + const std::string& cluster_id, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) = 0; +}; + +// Default no-op provider: behaves as if "no snapshot available". +class NoopSnapshotProvider final : public SnapshotProvider { + public: + bool LoadLatestSnapshot( + const std::string& /*cluster_id*/, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) override { + snapshot_id.clear(); + snapshot_sequence_id = 0; + snapshot.clear(); + return false; + } +}; + +} // namespace mooncake + + diff --git a/mooncake-store/include/standby_state_machine.h b/mooncake-store/include/standby_state_machine.h new file mode 100644 index 0000000000..649b462566 --- /dev/null +++ b/mooncake-store/include/standby_state_machine.h @@ -0,0 +1,334 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace mooncake { + +/** + * @brief Standby service states + * + * State transition diagram: + * + * ┌─────────┐ + * │ STOPPED │◄──────────────────────────────────────┐ + * └────┬────┘ │ + * │ Start() │ Stop()/Error + * ▼ │ + * ┌─────────────┐ │ + * │ CONNECTING │◄──────────────────────┐ │ + * └──────┬──────┘ │ │ + * │ Connected │ Reconnect │ + * ▼ │ │ + * ┌─────────────┐ Error/Gap ┌────┴─────┐ │ + * │ SYNCING │─────────────────►│RECOVERING│─────┤ + * └──────┬──────┘ └──────────┘ │ + * │ Sync complete │ + * ▼ │ + * ┌─────────────┐ Watch broken ┌────────────┐ │ + * │ WATCHING │─────────────────►│RECONNECTING│───┤ + * └──────┬──────┘ └────────────┘ │ + * │ Promote() │ + * ▼ │ + * ┌─────────────┐ │ + * │ PROMOTING │───────────────────────────────────┤ + * └──────┬──────┘ │ + * │ Success │ + * ▼ │ + * ┌─────────────┐ │ + * │ PROMOTED │───────────────────────────────────┘ + * └─────────────┘ + */ +enum class StandbyState : uint8_t { + // Initial state, service not started + STOPPED = 0, + + // Connecting to etcd cluster + CONNECTING = 1, + + // Initial sync: reading historical OpLog entries + SYNCING = 2, + + // Normal operation: watching for new OpLog entries + WATCHING = 3, + + // Recovering from error: re-syncing missed entries + RECOVERING = 4, + + // Reconnecting after watch failure + RECONNECTING = 5, + + // Promotion in progress: final catch-up before becoming Primary + PROMOTING = 6, + + // Successfully promoted to Primary + PROMOTED = 7, + + // Fatal error, cannot recover + FAILED = 8, +}; + +/** + * @brief Get human-readable state name + */ +inline const char* StandbyStateToString(StandbyState state) { + switch (state) { + case StandbyState::STOPPED: + return "STOPPED"; + case StandbyState::CONNECTING: + return "CONNECTING"; + case StandbyState::SYNCING: + return "SYNCING"; + case StandbyState::WATCHING: + return "WATCHING"; + case StandbyState::RECOVERING: + return "RECOVERING"; + case StandbyState::RECONNECTING: + return "RECONNECTING"; + case StandbyState::PROMOTING: + return "PROMOTING"; + case StandbyState::PROMOTED: + return "PROMOTED"; + case StandbyState::FAILED: + return "FAILED"; + default: + return "UNKNOWN"; + } +} + +/** + * @brief Events that trigger state transitions + */ +enum class StandbyEvent : uint8_t { + // User/system actions + START, // Start() called + STOP, // Stop() called + PROMOTE, // Promote() called + + // Connection events + CONNECTED, // Successfully connected to etcd + CONNECTION_FAILED, // Failed to connect to etcd + DISCONNECTED, // Connection lost + + // Sync events + SYNC_COMPLETE, // Initial sync completed + SYNC_FAILED, // Sync failed + + // Watch events + WATCH_HEALTHY, // Watch is healthy and receiving events + WATCH_BROKEN, // Watch connection broken + + // Recovery events + RECOVERY_SUCCESS, // Successfully recovered from error + RECOVERY_FAILED, // Recovery failed + + // Promotion events + PROMOTION_SUCCESS, // Successfully promoted + PROMOTION_FAILED, // Promotion failed + + // Error events + MAX_ERRORS_REACHED, // Too many consecutive errors + FATAL_ERROR, // Unrecoverable error +}; + +inline const char* StandbyEventToString(StandbyEvent event) { + switch (event) { + case StandbyEvent::START: + return "START"; + case StandbyEvent::STOP: + return "STOP"; + case StandbyEvent::PROMOTE: + return "PROMOTE"; + case StandbyEvent::CONNECTED: + return "CONNECTED"; + case StandbyEvent::CONNECTION_FAILED: + return "CONNECTION_FAILED"; + case StandbyEvent::DISCONNECTED: + return "DISCONNECTED"; + case StandbyEvent::SYNC_COMPLETE: + return "SYNC_COMPLETE"; + case StandbyEvent::SYNC_FAILED: + return "SYNC_FAILED"; + case StandbyEvent::WATCH_HEALTHY: + return "WATCH_HEALTHY"; + case StandbyEvent::WATCH_BROKEN: + return "WATCH_BROKEN"; + case StandbyEvent::RECOVERY_SUCCESS: + return "RECOVERY_SUCCESS"; + case StandbyEvent::RECOVERY_FAILED: + return "RECOVERY_FAILED"; + case StandbyEvent::PROMOTION_SUCCESS: + return "PROMOTION_SUCCESS"; + case StandbyEvent::PROMOTION_FAILED: + return "PROMOTION_FAILED"; + case StandbyEvent::MAX_ERRORS_REACHED: + return "MAX_ERRORS_REACHED"; + case StandbyEvent::FATAL_ERROR: + return "FATAL_ERROR"; + default: + return "UNKNOWN"; + } +} + +/** + * @brief State transition result + */ +struct StateTransitionResult { + bool allowed{false}; + StandbyState old_state{StandbyState::STOPPED}; + StandbyState new_state{StandbyState::STOPPED}; + std::string reason; +}; + +/** + * @brief Callback for state transition notifications + */ +using StateChangeCallback = + std::function; + +/** + * @brief Standby State Machine + * + * Thread-safe state machine for managing Standby service lifecycle. + * All state transitions are explicit and logged. + */ +class StandbyStateMachine { + public: + StandbyStateMachine(); + + /** + * @brief Get current state (thread-safe) + */ + StandbyState GetState() const { return current_state_.load(std::memory_order_acquire); } + + /** + * @brief Check if in a specific state + */ + bool IsInState(StandbyState state) const { return GetState() == state; } + + /** + * @brief Check if service is running (SYNCING, WATCHING, RECOVERING, RECONNECTING, + * PROMOTING) + */ + bool IsRunning() const { + StandbyState s = GetState(); + return s == StandbyState::SYNCING || s == StandbyState::WATCHING || + s == StandbyState::RECOVERING || s == StandbyState::RECONNECTING || + s == StandbyState::PROMOTING; + } + + /** + * @brief Check if connected to etcd + */ + bool IsConnected() const { + StandbyState s = GetState(); + return s == StandbyState::SYNCING || s == StandbyState::WATCHING || + s == StandbyState::RECOVERING || s == StandbyState::PROMOTING; + } + + /** + * @brief Check if watch is healthy + */ + bool IsWatchHealthy() const { return GetState() == StandbyState::WATCHING; } + + /** + * @brief Check if ready for promotion + */ + bool IsReadyForPromotion() const { return GetState() == StandbyState::WATCHING; } + + /** + * @brief Process an event and perform state transition + * @param event The event to process + * @return Result indicating if transition was allowed and new state + */ + StateTransitionResult ProcessEvent(StandbyEvent event); + + /** + * @brief Register a callback for state change notifications + */ + void RegisterCallback(StateChangeCallback callback); + + /** + * @brief State transition record for debugging + */ + struct TransitionRecord { + std::chrono::steady_clock::time_point timestamp; + StandbyState from_state; + StandbyState to_state; + StandbyEvent event; + }; + + /** + * @brief Get state transition history (for debugging) + */ + std::vector GetTransitionHistory(size_t max_records = 100) const; + + /** + * @brief Get time spent in current state + */ + std::chrono::milliseconds GetTimeInCurrentState() const; + + /** + * @brief Get consecutive error count + */ + int GetConsecutiveErrors() const { return consecutive_errors_.load(); } + + /** + * @brief Increment consecutive error count + * @return New error count + */ + int IncrementErrors(); + + /** + * @brief Reset consecutive error count + */ + void ResetErrors() { consecutive_errors_.store(0); } + + /** + * @brief Get reconnect attempt count + */ + int GetReconnectCount() const { return reconnect_count_.load(); } + + /** + * @brief Increment reconnect count + */ + void IncrementReconnectCount() { reconnect_count_.fetch_add(1); } + + /** + * @brief Reset reconnect count + */ + void ResetReconnectCount() { reconnect_count_.store(0); } + + // Constants + static constexpr int kMaxConsecutiveErrors = 10; + static constexpr int kMaxReconnectAttempts = 100; + + private: + /** + * @brief Check if a transition is valid and get new state + */ + StateTransitionResult ValidateTransition(StandbyState from, StandbyEvent event) const; + + /** + * @brief Notify all registered callbacks + */ + void NotifyCallbacks(StandbyState old_state, StandbyState new_state, StandbyEvent event); + + std::atomic current_state_{StandbyState::STOPPED}; + std::atomic consecutive_errors_{0}; + std::atomic reconnect_count_{0}; + std::chrono::steady_clock::time_point state_enter_time_; + + mutable std::mutex mutex_; + std::vector callbacks_; + std::vector transition_history_; + + static constexpr size_t kMaxHistorySize = 1000; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/types.h b/mooncake-store/include/types.h index 8fbdb806bc..0a21a3d80d 100644 --- a/mooncake-store/include/types.h +++ b/mooncake-store/include/types.h @@ -23,6 +23,65 @@ namespace mooncake { static constexpr uint64_t WRONG_VERSION = 0; static constexpr uint64_t DEFAULT_VALUE = UINT64_MAX; static constexpr uint64_t ERRNO_BASE = DEFAULT_VALUE - 1000; + +// Sequence ID comparison utilities for wrap-around safety. +// These functions use signed difference to correctly handle uint64_t overflow +// (from UINT64_MAX wrapping to 0). Assumes sequence IDs won't differ by more +// than 2^63, which is reasonable for practical systems. +// +// Example: If sequence_id wraps from UINT64_MAX to 0, then: +// IsSequenceNewer(0, UINT64_MAX) = true (0 is newer after wrap) +// IsSequenceNewer(UINT64_MAX, 0) = false (UINT64_MAX is older before wrap) +// +// Note: We use 'static inline' here to give these small helpers internal +// linkage and avoid any potential ODR/linkage issues in large codebases. +static inline bool IsSequenceNewer(uint64_t a, uint64_t b) { + // Cast to int64_t to get signed difference, then check if positive. + // This correctly handles wrap-around: if a wrapped from UINT64_MAX to 0, + // then (int64_t)(a - b) will be positive (assuming gap < 2^63). + return static_cast(a - b) > 0; +} + +static inline bool IsSequenceOlder(uint64_t a, uint64_t b) { + return static_cast(a - b) < 0; +} + +static inline bool IsSequenceEqual(uint64_t a, uint64_t b) { + return a == b; +} + +static inline bool IsSequenceNewerOrEqual(uint64_t a, uint64_t b) { + return a == b || static_cast(a - b) > 0; +} + +static inline bool IsSequenceOlderOrEqual(uint64_t a, uint64_t b) { + return a == b || static_cast(a - b) < 0; +} + +// Cluster ID validation utilities. +// +// cluster_id is used to construct etcd key prefixes (e.g. "/oplog//..."). +// To avoid key-prefix injection / accidental cross-cluster overlap, we restrict the +// allowed characters to a conservative safe set. We validate the "component" form +// (without trailing slash). Trailing slashes should be normalized away before +// validation. +static inline bool IsValidClusterIdComponent(const std::string& cluster_id) { + if (cluster_id.empty()) { + return false; + } + if (cluster_id.size() > 128) { + return false; + } + for (unsigned char c : cluster_id) { + const bool ok = + (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || c == '_' || c == '-' || c == '.'; + if (!ok) { + return false; + } + } + return true; +} static constexpr uint64_t DEFAULT_DEFAULT_KV_LEASE_TTL = 5000; // in milliseconds static constexpr uint64_t DEFAULT_KV_SOFT_PIN_TTL_MS = diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 05968b56df..7d4976e891 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -16,8 +16,6 @@ set(MOONCAKE_STORE_SOURCES ha_helper.cpp segment.cpp transfer_task.cpp - etcd_helper.cpp - ha_helper.cpp rpc_service.cpp offset_allocator.cpp posix_file.cpp @@ -26,10 +24,28 @@ set(MOONCAKE_STORE_SOURCES dummy_client.cpp http_metadata_server.cpp file_storage.cpp + oplog_manager.cpp + etcd_oplog_store.cpp + oplog_watcher.cpp + oplog_applier.cpp + hot_standby_service.cpp + standby_state_machine.cpp + ha_metric_manager.cpp + # replication_service.cpp removed - using etcd-based OpLog sync instead ) set(EXTRA_LIBS "") +# Find xxHash (required for ComputeChecksum) +find_path(XXHASH_INCLUDE_DIR NAMES xxhash.h PATHS /usr/include /usr/local/include) +find_library(XXHASH_LIBRARY NAMES xxhash libxxhash PATHS /usr/lib /usr/local/lib /usr/lib64) +if (XXHASH_INCLUDE_DIR AND XXHASH_LIBRARY) + message(STATUS "Found xxHash: include=${XXHASH_INCLUDE_DIR} lib=${XXHASH_LIBRARY}") + list(APPEND MASTER_EXTRA_INCS ${XXHASH_INCLUDE_DIR}) +else() + message(FATAL_ERROR "xxHash library/header not found. Please install xxhash (development headers) and try again.") +endif() + if(USE_3FS) add_subdirectory(hf3fs) list(APPEND MOONCAKE_STORE_SOURCES ${HF3FS_SOURCES}) @@ -43,6 +59,8 @@ endif() # The cache_allocator library include_directories(${Python3_INCLUDE_DIRS}) add_library(mooncake_store ${MOONCAKE_STORE_SOURCES}) +target_include_directories(mooncake_store PUBLIC ${XXHASH_INCLUDE_DIR}) +target_link_libraries(mooncake_store PUBLIC ${XXHASH_LIBRARY}) target_link_libraries(mooncake_store PUBLIC transfer_engine cachelib_memory_allocator ${ETCD_WRAPPER_LIB} glog::glog gflags::gflags ${EXTRA_LIBS} ) diff --git a/mooncake-store/src/client_service.cpp b/mooncake-store/src/client_service.cpp index c62789b634..0193c22e07 100644 --- a/mooncake-store/src/client_service.cpp +++ b/mooncake-store/src/client_service.cpp @@ -175,7 +175,30 @@ tl::expected CheckRegisterMemoryParams(const void* addr, ErrorCode Client::ConnectToMaster(const std::string& master_server_entry) { if (master_server_entry.find("etcd://") == 0) { + // Support optional cluster_id in connection string: + // etcd://?cluster_id= + // If not provided, MasterViewHelper will fall back to env MC_STORE_CLUSTER_ID, + // then DEFAULT_CLUSTER_ID. std::string etcd_entry = master_server_entry.substr(strlen("etcd://")); + std::string cluster_id; + { + const size_t qpos = etcd_entry.find('?'); + if (qpos != std::string::npos) { + const std::string query = etcd_entry.substr(qpos + 1); + etcd_entry = etcd_entry.substr(0, qpos); + const std::string k = "cluster_id="; + const size_t kpos = query.find(k); + if (kpos != std::string::npos) { + const size_t vpos = kpos + k.size(); + size_t vend = query.find('&', vpos); + if (vend == std::string::npos) vend = query.size(); + cluster_id = query.substr(vpos, vend - vpos); + } + } + } + if (!cluster_id.empty()) { + master_view_helper_.SetClusterId(cluster_id); + } // Get master address from etcd auto err = master_view_helper_.ConnectToEtcd(etcd_entry); diff --git a/mooncake-store/src/etcd_helper.cpp b/mooncake-store/src/etcd_helper.cpp index 5417de5afc..ae3957a2f9 100644 --- a/mooncake-store/src/etcd_helper.cpp +++ b/mooncake-store/src/etcd_helper.cpp @@ -152,6 +152,194 @@ ErrorCode EtcdHelper::CancelKeepAlive(EtcdLeaseId lease_id) { } return ErrorCode::OK; } + +ErrorCode EtcdHelper::Put(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + char* err_msg = nullptr; + int ret = EtcdStorePutWrapper((char*)key, (int)key_size, (char*)value, + (int)value_size, &err_msg); + if (ret != 0) { + LOG(ERROR) << "key=" << std::string(key, key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::Create(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + char* err_msg = nullptr; + int ret = EtcdStoreCreateWrapper((char*)key, (int)key_size, (char*)value, + (int)value_size, &err_msg); + if (ret == -2) { + free(err_msg); + return ErrorCode::ETCD_TRANSACTION_FAIL; + } + if (ret != 0) { + LOG(ERROR) << "key=" << std::string(key, key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values) { + // TODO: Implement GetWithPrefix - need to simplify Go wrapper interface first + // For now, return error as this requires complex memory management + LOG(ERROR) << "GetWithPrefix not yet implemented - requires Go wrapper interface simplification"; + return ErrorCode::INTERNAL_ERROR; +} + +ErrorCode EtcdHelper::GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id) { + char* err_msg = nullptr; + char* json_ptr = nullptr; + int json_size = 0; + // Go wrapper takes int limit. + int ret = EtcdStoreGetRangeAsJsonWrapper((char*)start_key, (int)start_key_size, + (char*)end_key, (int)end_key_size, + (int)limit, &json_ptr, &json_size, + (GoInt64*)&revision_id, &err_msg); + if (ret != 0) { + LOG(ERROR) << "start_key=" << std::string(start_key, start_key_size) + << ", end_key=" << std::string(end_key, end_key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + json = std::string(json_ptr, json_size); + free(json_ptr); + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key) { + char* err_msg = nullptr; + char* first_key_ptr = nullptr; + int first_key_size = 0; + int ret = EtcdStoreGetFirstKeyWithPrefixWrapper((char*)prefix, (int)prefix_size, + &first_key_ptr, &first_key_size, + &err_msg); + if (ret == -2) { + free(err_msg); + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + first_key = std::string(first_key_ptr, first_key_size); + free(first_key_ptr); + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::GetLastKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& last_key) { + char* err_msg = nullptr; + char* last_key_ptr = nullptr; + int last_key_size = 0; + int ret = EtcdStoreGetLastKeyWithPrefixWrapper((char*)prefix, (int)prefix_size, + &last_key_ptr, &last_key_size, + &err_msg); + if (ret == -2) { + free(err_msg); + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + last_key = std::string(last_key_ptr, last_key_size); + free(last_key_ptr); + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size) { + char* err_msg = nullptr; + int ret = EtcdStoreDeleteRangeWrapper((char*)start_key, (int)start_key_size, + (char*)end_key, (int)end_key_size, + &err_msg); + if (ret != 0) { + LOG(ERROR) << "start_key=" << std::string(start_key, start_key_size) + << ", end_key=" << std::string(end_key, end_key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::WatchWithPrefix(const char* prefix, const size_t prefix_size, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, + const char*, size_t, int)) { + char* err_msg = nullptr; + // Convert function pointer to void* for passing to Go function + // Note: This is safe because we're just passing the pointer, not calling it + void* callback_func_ptr = reinterpret_cast(callback_func); + int ret = EtcdStoreWatchWithPrefixWrapper((char*)prefix, (int)prefix_size, + callback_context, callback_func_ptr, + &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)) { + char* err_msg = nullptr; + void* callback_func_ptr = reinterpret_cast(callback_func); + int ret = EtcdStoreWatchWithPrefixFromRevisionV2Wrapper( + (char*)prefix, (int)prefix_size, (GoInt64)start_revision, callback_context, + callback_func_ptr, &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", start_revision=" << (int64_t)start_revision + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size) { + char* err_msg = nullptr; + int ret = EtcdStoreCancelWatchWithPrefixWrapper((char*)prefix, (int)prefix_size, + &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} #else ErrorCode EtcdHelper::ConnectToEtcdStoreClient( const std::string& etcd_endpoints) { @@ -200,6 +388,99 @@ ErrorCode EtcdHelper::CancelKeepAlive(EtcdLeaseId lease_id) { return ErrorCode::ETCD_OPERATION_ERROR; } +ErrorCode EtcdHelper::Put(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::Create(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + (void)key; + (void)key_size; + (void)value; + (void)value_size; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id) { + (void)start_key; + (void)start_key_size; + (void)end_key; + (void)end_key_size; + (void)limit; + (void)json; + (void)revision_id; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} +ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::GetLastKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& last_key) { + (void)prefix; + (void)prefix_size; + (void)last_key; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::WatchWithPrefix(const char* prefix, const size_t prefix_size, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, + const char*, size_t, int)) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)) { + (void)prefix; + (void)prefix_size; + (void)start_revision; + (void)callback_context; + (void)callback_func; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + #endif } // namespace mooncake \ No newline at end of file diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp new file mode 100644 index 0000000000..6d5530a172 --- /dev/null +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -0,0 +1,508 @@ +#include "etcd_oplog_store.h" + +#include +#include +#include + +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif + +#include "etcd_helper.h" + +namespace mooncake { + +EtcdOpLogStore::EtcdOpLogStore(const std::string& cluster_id, + bool enable_latest_seq_batch_update) + : cluster_id_(cluster_id), + enable_latest_seq_batch_update_(enable_latest_seq_batch_update), + last_update_time_(std::chrono::steady_clock::now()) { + // Normalize cluster_id to avoid accidental double slashes in etcd keys when + // caller passes a trailing '/' (master_view_key uses trailing '/', OpLog keys don't). + while (!cluster_id_.empty() && cluster_id_.back() == '/') { + cluster_id_.pop_back(); + } + + if (!cluster_id_.empty() && !IsValidClusterIdComponent(cluster_id_)) { + LOG(FATAL) << "Invalid cluster_id for EtcdOpLogStore: '" << cluster_id_ + << "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } + + // Start batch update thread only for writers. + if (enable_latest_seq_batch_update_) { + batch_update_running_.store(true); + batch_update_thread_ = + std::thread(&EtcdOpLogStore::BatchUpdateThread, this); + } +} + +EtcdOpLogStore::~EtcdOpLogStore() { + if (!enable_latest_seq_batch_update_) { + return; + } + + // Stop batch update thread + batch_update_running_.store(false); + if (batch_update_thread_.joinable()) { + batch_update_thread_.join(); + } + + // Perform final update if there are pending updates + if (pending_count_.load() > 0) { + DoBatchUpdate(); + } +} + +ErrorCode EtcdOpLogStore::WriteOpLog(const OpLogEntry& entry) { + std::string key = BuildOpLogKey(entry.sequence_id); + std::string value = SerializeOpLogEntry(entry); + + // Fence: never overwrite an existing OpLog key. + // - If this is a retry of the same entry: key exists with same value => OK. + // - If key exists with different value: conflict => error (signals seq regression / bug). + ErrorCode err = EtcdHelper::Create(key.c_str(), key.size(), value.c_str(), value.size()); + if (err == ErrorCode::ETCD_TRANSACTION_FAIL) { + std::string existing; + EtcdRevisionId rev = 0; + ErrorCode get_err = EtcdHelper::Get(key.c_str(), key.size(), existing, rev); + if (get_err == ErrorCode::OK && existing == value) { + // Idempotent retry. + err = ErrorCode::OK; + } else { + LOG(ERROR) << "OpLog key conflict: seq=" << entry.sequence_id + << ", get_err=" << get_err; + return ErrorCode::ETCD_OPERATION_ERROR; + } + } + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to write OpLog entry, sequence_id=" << entry.sequence_id; + return err; + } + + // Update `/latest`. + // - Writers: batch update to reduce etcd write pressure. + // - Readers / tests: update immediately for simplicity. + if (!enable_latest_seq_batch_update_) { + return UpdateLatestSequenceId(entry.sequence_id); + } + + pending_latest_seq_id_.store(entry.sequence_id); + size_t count = pending_count_.fetch_add(1) + 1; + if (count >= kBatchSize) { + DoBatchUpdate(); + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::ReadOpLog(uint64_t sequence_id, + OpLogEntry& entry) { + std::string key = BuildOpLogKey(sequence_id); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry, sequence_id=" + << sequence_id; + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::ReadOpLogSince(uint64_t start_sequence_id, + size_t limit, + std::vector& entries) { + EtcdRevisionId rev = 0; + return ReadOpLogSinceWithRevision(start_sequence_id, limit, entries, rev); +} + +ErrorCode EtcdOpLogStore::ReadOpLogSinceWithRevision(uint64_t start_sequence_id, + size_t limit, + std::vector& entries, + EtcdRevisionId& revision_id) { + entries.clear(); + entries.reserve(limit); + + // Range is limited to OpLog entry keys only. + const std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/"; + std::string current_start_key = BuildOpLogKey(start_sequence_id + 1); + + // Compute prefix range end (etcd prefix end). + auto prefix_end = [](std::string p) -> std::string { + for (int i = static_cast(p.size()) - 1; i >= 0; --i) { + unsigned char c = static_cast(p[i]); + if (c < 0xFF) { + p[i] = static_cast(c + 1); + p.resize(i + 1); + return p; + } + } + return std::string(1, '\0'); + }; + const std::string end_key = prefix_end(prefix); + + // Pagination: + // - Use range-get with limit + // - Start next page from lastKey + '\0' (lexicographically just after lastKey) + // This avoids repeating the last key without adding new Go/C++ APIs. + revision_id = 0; + while (entries.size() < limit) { + const size_t page_limit = limit - entries.size(); + std::string json; + EtcdRevisionId page_rev = 0; + ErrorCode err = + EtcdHelper::GetRangeAsJson(current_start_key.c_str(), + current_start_key.size(), end_key.c_str(), + end_key.size(), page_limit, json, page_rev); + if (err != ErrorCode::OK) { + return err; + } + if (page_rev > revision_id) { + revision_id = page_rev; + } + + // Parse kv list: [{"key":"...","value":"..."}] + Json::Value root; + Json::CharReaderBuilder reader; + std::string errs; + std::istringstream s(json); + if (!Json::parseFromStream(reader, s, &root, &errs)) { + LOG(ERROR) << "Failed to parse range JSON: " << errs; + return ErrorCode::INTERNAL_ERROR; + } + if (!root.isArray()) { + return ErrorCode::INTERNAL_ERROR; + } + if (root.empty()) { + break; // no more data + } + + std::string last_key_in_page; + for (const auto& kv : root) { + const std::string key = kv.get("key", "").asString(); + last_key_in_page = key; + if (key.empty() || key.find("/latest") != std::string::npos || + key.find("/snapshot/") != std::string::npos) { + continue; + } + + // Parse seq from key suffix and filter (handles legacy keys too). + size_t pos = key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= key.size()) { + continue; + } + uint64_t seq = 0; + try { + seq = static_cast(std::stoull(key.substr(pos + 1))); + } catch (...) { + continue; + } + if (IsSequenceOlderOrEqual(seq, start_sequence_id)) { + continue; + } + + OpLogEntry entry; + const std::string value = kv.get("value", "").asString(); + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry from key=" << key; + return ErrorCode::INTERNAL_ERROR; + } + entries.push_back(std::move(entry)); + if (entries.size() >= limit) { + break; + } + } + + // Advance start key for next page. + if (last_key_in_page.empty()) { + break; + } + current_start_key = last_key_in_page; + current_start_key.push_back('\0'); + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::GetLatestSequenceId(uint64_t& sequence_id) { + std::string key = BuildLatestKey(); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + try { + sequence_id = std::stoull(value); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse latest sequence_id: " << e.what(); + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::GetMaxSequenceId(uint64_t& sequence_id) { + auto max_seq_opt = GetMaxSequenceIdInternal(); + if (!max_seq_opt.has_value()) { + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + sequence_id = max_seq_opt.value(); + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::UpdateLatestSequenceId(uint64_t sequence_id) { + std::string key = BuildLatestKey(); + std::string value = std::to_string(sequence_id); + return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), value.size()); +} + +ErrorCode EtcdOpLogStore::RecordSnapshotSequenceId( + const std::string& snapshot_id, uint64_t sequence_id) { + std::string key = BuildSnapshotKey(snapshot_id); + std::string value = std::to_string(sequence_id); + return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), value.size()); +} + +ErrorCode EtcdOpLogStore::GetSnapshotSequenceId( + const std::string& snapshot_id, uint64_t& sequence_id) { + std::string key = BuildSnapshotKey(snapshot_id); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + try { + sequence_id = std::stoull(value); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse snapshot sequence_id: " << e.what(); + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::CleanupOpLogBefore(uint64_t before_sequence_id) { + // Robust cleanup (Scheme 3): + // - Determine current minimum sequence_id in etcd + // - DeleteRange [min_key, before_key) + // + // IMPORTANT: This relies on lexicographical ordering of keys, so the + // sequence_id portion MUST be fixed-width (zero-padded). + auto min_seq_opt = GetMinSequenceId(); + if (!min_seq_opt.has_value()) { + return ErrorCode::OK; // nothing to cleanup + } + + uint64_t min_seq = min_seq_opt.value(); + if (before_sequence_id <= min_seq) { + return ErrorCode::OK; + } + + std::string start_key = BuildOpLogKey(min_seq); + std::string end_key = BuildOpLogKey(before_sequence_id); // delete < before_sequence_id + + return EtcdHelper::DeleteRange(start_key.c_str(), start_key.size(), + end_key.c_str(), end_key.size()); +} + +std::string EtcdOpLogStore::BuildOpLogKey(uint64_t sequence_id) const { + std::ostringstream oss; + // Fixed-width encoding for correct etcd lexicographical range operations. + // 20 digits is enough for uint64_t max (18446744073709551615). + oss << kOpLogPrefix << cluster_id_ << "/" + << std::setw(20) << std::setfill('0') << sequence_id; + return oss.str(); +} + +std::optional EtcdOpLogStore::GetMinSequenceId() const { + std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/"; + std::string first_key; + ErrorCode err = + EtcdHelper::GetFirstKeyWithPrefix(prefix.c_str(), prefix.size(), first_key); + if (err != ErrorCode::OK) { + return std::nullopt; + } + + // Skip non-entry keys if any (e.g. "/latest" or "/snapshot/..."). + // Entries are expected to be ".../<20-digit-seq>". + // If the first key isn't an entry key, fall back to nullopt (safe no-op). + if (first_key.find("/latest") != std::string::npos || + first_key.find("/snapshot/") != std::string::npos) { + return std::nullopt; + } + + size_t pos = first_key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= first_key.size()) { + return std::nullopt; + } + std::string seq_str = first_key.substr(pos + 1); + try { + return static_cast(std::stoull(seq_str)); + } catch (...) { + return std::nullopt; + } +} + +std::optional EtcdOpLogStore::GetMaxSequenceIdInternal() const { + // Entry keys are fixed-width 20-digit numbers, which (in practice) start with '0'. + // Use "/0" to avoid picking up "/latest" which is lexicographically after digits. + std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/0"; + std::string last_key; + ErrorCode err = + EtcdHelper::GetLastKeyWithPrefix(prefix.c_str(), prefix.size(), last_key); + if (err != ErrorCode::OK) { + return std::nullopt; + } + + size_t pos = last_key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= last_key.size()) { + return std::nullopt; + } + std::string seq_str = last_key.substr(pos + 1); + try { + return static_cast(std::stoull(seq_str)); + } catch (...) { + return std::nullopt; + } +} + +std::string EtcdOpLogStore::BuildLatestKey() const { + std::ostringstream oss; + oss << kOpLogPrefix << cluster_id_ << kLatestSuffix; + return oss.str(); +} + +std::string EtcdOpLogStore::BuildSnapshotKey( + const std::string& snapshot_id) const { + std::ostringstream oss; + oss << kOpLogPrefix << cluster_id_ << kSnapshotSuffix << snapshot_id + << "/sequence_id"; + return oss.str(); +} + +std::string EtcdOpLogStore::SerializeOpLogEntry( + const OpLogEntry& entry) const { + Json::Value root; + root["sequence_id"] = static_cast(entry.sequence_id); + root["timestamp_ms"] = static_cast(entry.timestamp_ms); + root["op_type"] = static_cast(entry.op_type); + root["object_key"] = entry.object_key; + root["payload"] = entry.payload; + root["checksum"] = static_cast(entry.checksum); + root["prefix_hash"] = static_cast(entry.prefix_hash); + + Json::StreamWriterBuilder builder; + builder["indentation"] = ""; // Compact format + std::unique_ptr writer(builder.newStreamWriter()); + std::ostringstream oss; + writer->write(root, &oss); + return oss.str(); +} + +bool EtcdOpLogStore::DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) const { + Json::Value root; + Json::CharReaderBuilder builder; + std::unique_ptr reader(builder.newCharReader()); + std::string errors; + + if (!reader->parse(json_str.data(), json_str.data() + json_str.size(), + &root, &errors)) { + LOG(ERROR) << "Failed to parse JSON: " << errors; + return false; + } + + try { + entry.sequence_id = root["sequence_id"].asUInt64(); + entry.timestamp_ms = root["timestamp_ms"].asUInt64(); + entry.op_type = static_cast(root["op_type"].asInt()); + entry.object_key = root["object_key"].asString(); + entry.payload = root["payload"].asString(); + entry.checksum = root["checksum"].asUInt(); + entry.prefix_hash = root["prefix_hash"].asUInt(); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to deserialize OpLogEntry: " << e.what(); + return false; + } + + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "EtcdOpLogStore: entry size rejected, sequence_id=" + << entry.sequence_id << ", key=" << entry.object_key + << ", reason=" << size_reason; + return false; + } + + return true; +} + +void EtcdOpLogStore::BatchUpdateThread() { + if (!enable_latest_seq_batch_update_) { + return; + } + while (batch_update_running_.load()) { + std::this_thread::sleep_for( + std::chrono::milliseconds(kBatchIntervalMs)); + + // Check if we need to update based on time interval + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast( + now - last_update_time_).count(); + + if (pending_count_.load() > 0 && elapsed >= kBatchIntervalMs) { + DoBatchUpdate(); + } + } +} + +void EtcdOpLogStore::TriggerBatchUpdateIfNeeded() { + // This method is kept for potential future use (e.g., manual trigger) + // Currently, DoBatchUpdate() is called directly from WriteOpLog + // when batch size threshold is reached + if (pending_count_.load() >= kBatchSize) { + DoBatchUpdate(); + } +} + +void EtcdOpLogStore::DoBatchUpdate() { + if (!enable_latest_seq_batch_update_) { + return; + } + std::lock_guard lock(batch_update_mutex_); + + // Get the pending sequence_id and reset counters + uint64_t seq_id_to_update = pending_latest_seq_id_.load(); + size_t count = pending_count_.exchange(0); + + if (count == 0) { + return; // Nothing to update + } + + // Update latest_sequence_id in etcd + ErrorCode err = UpdateLatestSequenceId(seq_id_to_update); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to batch update latest_sequence_id=" + << seq_id_to_update << ", error=" << err + << ". Will retry in next batch."; + // Restore the count so it will be retried + pending_count_.fetch_add(count); + } else { + last_update_time_ = std::chrono::steady_clock::now(); + VLOG(2) << "Batch updated latest_sequence_id=" << seq_id_to_update + << " (count=" << count << " entries)"; + } +} + +} // namespace mooncake + diff --git a/mooncake-store/src/ha_helper.cpp b/mooncake-store/src/ha_helper.cpp index 07906772af..09709a5978 100644 --- a/mooncake-store/src/ha_helper.cpp +++ b/mooncake-store/src/ha_helper.cpp @@ -1,16 +1,64 @@ #include "ha_helper.h" + +#include + +#include +#include + #include "etcd_helper.h" +#include "hot_standby_service.h" #include "rpc_service.h" namespace mooncake { -MasterViewHelper::MasterViewHelper() { - std::string cluster_id; - const char* cluster_id_env = std::getenv("MC_STORE_CLUSTER_ID"); - if (cluster_id_env != nullptr && strlen(cluster_id_env) > 0) { - cluster_id = cluster_id_env; +namespace { +std::string ResolveClusterIdForMasterView(const std::string& cluster_id) { + std::string resolved; + if (!cluster_id.empty()) { + resolved = cluster_id; } else { - cluster_id = "mooncake"; + const char* cluster_id_env = std::getenv("MC_STORE_CLUSTER_ID"); + if (cluster_id_env != nullptr && strlen(cluster_id_env) > 0) { + resolved = std::string(cluster_id_env); + } else { + resolved = DEFAULT_CLUSTER_ID; + } + } + + // Validate resolved cluster_id (even if it's the default). + // Strip trailing slashes for validation. + std::string normalized = resolved; + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + if (!normalized.empty() && !IsValidClusterIdComponent(normalized)) { + LOG(FATAL) << "Invalid cluster_id resolved for MasterViewHelper: '" << resolved + << "' (normalized: '" << normalized + << "'). Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } + + return resolved; +} +} // namespace + +MasterViewHelper::MasterViewHelper(const std::string& cluster_id) { + BuildMasterViewKeyFromClusterId(ResolveClusterIdForMasterView(cluster_id)); +} + +void MasterViewHelper::SetClusterId(const std::string& cluster_id) { + BuildMasterViewKeyFromClusterId(ResolveClusterIdForMasterView(cluster_id)); +} + +void MasterViewHelper::BuildMasterViewKeyFromClusterId( + const std::string& cluster_id_in) { + std::string cluster_id = cluster_id_in; + // Normalize cluster_id for validation: strip trailing slashes. + while (!cluster_id.empty() && cluster_id.back() == '/') { + cluster_id.pop_back(); + } + if (!IsValidClusterIdComponent(cluster_id)) { + LOG(FATAL) << "Invalid cluster_id for MasterViewHelper: '" << cluster_id + << "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; } // Ensure the cluster_id ends with '/' if not empty if (!cluster_id.empty() && cluster_id.back() != '/') { @@ -125,19 +173,77 @@ int MasterServiceSupervisor::Start() { } LOG(INFO) << "Init leader election helper..."; - MasterViewHelper mv_helper; + MasterViewHelper mv_helper(config_.cluster_id); if (mv_helper.ConnectToEtcd(config_.etcd_endpoints) != ErrorCode::OK) { LOG(ERROR) << "Failed to connect to etcd endpoints: " << config_.etcd_endpoints; return -1; } - LOG(INFO) << "Trying to elect self as leader..."; + +#ifdef STORE_USE_ETCD + // Connect to etcd for OpLog sync + if (EtcdHelper::ConnectToEtcdStoreClient(config_.etcd_endpoints.c_str()) != + ErrorCode::OK) { + LOG(ERROR) << "Failed to connect to etcd store client: " + << config_.etcd_endpoints; + return -1; + } +#endif + + LOG(INFO) << "Checking for existing leader..."; EtcdLeaseId lease_id = 0; - // view_version will be updated by ElectLeader and then used in - // WrappedMasterService ViewVersionId view_version = 0; + + // Check if there is already a leader + std::string current_leader; + ViewVersionId current_version = 0; + auto ret = mv_helper.GetMasterView(current_leader, current_version); + bool had_standby = false; + + if (ret == ErrorCode::OK) { + // There is an existing leader, start Standby service + LOG(INFO) << "Found existing leader: " << current_leader + << ", starting Standby service..."; + StartStandbyService(mv_helper, current_leader); + had_standby = true; + + // Watch until leader is deleted + LOG(INFO) << "Watching for leadership change..."; + auto watch_ret = EtcdHelper::WatchUntilDeleted( + mv_helper.GetMasterViewKey().c_str(), mv_helper.GetMasterViewKey().size()); + + if (watch_ret != ErrorCode::OK) { + LOG(ERROR) << "Error watching for leadership change: " << watch_ret; + // Stop Standby service on watch error and retry. + StopStandbyService(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + LOG(INFO) << "Leader disappeared, trying to elect self as leader..."; + } else { + LOG(INFO) << "No existing leader found, trying to elect self as leader..."; + } + + // Try to elect self as leader mv_helper.ElectLeader(config_.local_hostname, view_version, lease_id); + // If we were running as Standby, finalize catch-up and snapshot metadata now. + std::vector> standby_snapshot; + uint64_t standby_last_seq_id = 0; +#ifdef STORE_USE_ETCD + if (had_standby && standby_service_ && standby_running_.load()) { + LOG(INFO) << "Finalizing standby state for promotion..."; + standby_service_->Promote(); // does final catch-up sync + stops watcher + standby_last_seq_id = standby_service_->GetLatestAppliedSequenceId(); + standby_service_->ExportMetadataSnapshot(standby_snapshot); + // We are now leader; standby service is no longer needed. + StopStandbyService(); + LOG(INFO) << "Standby snapshot ready: keys=" << standby_snapshot.size() + << ", last_seq_id=" << standby_last_seq_id; + } +#endif + // Start a thread to keep the leader alive auto keep_leader_thread = std::thread([&server, &mv_helper, lease_id]() { @@ -154,6 +260,14 @@ int MasterServiceSupervisor::Start() { LOG(INFO) << "Starting master service..."; mooncake::WrappedMasterService wrapped_master_service( mooncake::WrappedMasterServiceConfig(config_, view_version)); + + // Restore from promoted standby snapshot if available. +#ifdef STORE_USE_ETCD + if (standby_last_seq_id > 0 || !standby_snapshot.empty()) { + wrapped_master_service.RestoreFromStandby(standby_snapshot, standby_last_seq_id); + } +#endif + mooncake::RegisterRpcService(server, wrapped_master_service); // Metric reporting is now handled by WrappedMasterService. @@ -186,7 +300,56 @@ int MasterServiceSupervisor::Start() { return 0; } +void MasterServiceSupervisor::StartStandbyService(MasterViewHelper& mv_helper, + const std::string& current_leader) { +#ifdef STORE_USE_ETCD + if (standby_running_.load()) { + LOG(WARNING) << "Standby service is already running"; + return; + } + + HotStandbyConfig standby_config; + standby_config.standby_id = config_.local_hostname; + standby_config.primary_address = current_leader; + standby_config.verification_interval_sec = 30; + standby_config.max_replication_lag_entries = 1000; + standby_config.enable_verification = false; // Disable verification for now + + standby_service_ = std::make_unique(standby_config); + + ErrorCode err = standby_service_->Start( + current_leader, config_.etcd_endpoints, config_.cluster_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start Standby service: " << err; + standby_service_.reset(); + return; + } + + standby_running_.store(true); + LOG(INFO) << "Standby service started successfully"; +#else + LOG(WARNING) << "STORE_USE_ETCD is not enabled, cannot start Standby service"; +#endif +} + +void MasterServiceSupervisor::StopStandbyService() { +#ifdef STORE_USE_ETCD + if (!standby_running_.load()) { + return; + } + + if (standby_service_) { + standby_service_->Stop(); + standby_service_.reset(); + } + + standby_running_.store(false); + LOG(INFO) << "Standby service stopped"; +#endif +} + MasterServiceSupervisor::~MasterServiceSupervisor() { + StopStandbyService(); if (server_thread_.joinable()) { server_thread_.join(); } diff --git a/mooncake-store/src/ha_metric_manager.cpp b/mooncake-store/src/ha_metric_manager.cpp new file mode 100644 index 0000000000..b16c007aca --- /dev/null +++ b/mooncake-store/src/ha_metric_manager.cpp @@ -0,0 +1,281 @@ +#include "ha_metric_manager.h" + +#include + +#include +#include + +namespace mooncake { + +// --- Singleton Instance --- +HAMetricManager& HAMetricManager::instance() { + static HAMetricManager static_instance; + return static_instance; +} + +// --- Constructor --- +HAMetricManager::HAMetricManager() + // OpLog Sequence Gauges + : oplog_last_sequence_id_( + "ha_oplog_last_sequence_id", + "Latest OpLog sequence ID written by Primary"), + oplog_applied_sequence_id_( + "ha_oplog_applied_sequence_id", + "Latest OpLog sequence ID applied by Standby"), + oplog_standby_lag_( + "ha_oplog_standby_lag", + "Number of OpLog entries Standby is behind Primary"), + oplog_pending_entries_( + "ha_oplog_pending_entries", + "Number of out-of-order entries waiting in OpLogApplier"), + pending_mutation_queue_size_( + "ha_pending_mutation_queue_size", + "Number of mutations pending etcd write retry"), + + // Error Counters + oplog_skipped_entries_total_( + "ha_oplog_skipped_entries_total", + "Total number of OpLog entries skipped due to timeout"), + oplog_checksum_failures_total_( + "ha_oplog_checksum_failures_total", + "Total number of OpLog entries with checksum verification failures"), + oplog_gap_resolve_attempts_total_( + "ha_oplog_gap_resolve_attempts_total", + "Total number of attempts to resolve missing OpLog entries"), + oplog_gap_resolve_success_total_( + "ha_oplog_gap_resolve_success_total", + "Total number of successfully resolved missing OpLog entries"), + oplog_etcd_write_failures_total_( + "ha_oplog_etcd_write_failures_total", + "Total number of failed etcd write operations"), + oplog_etcd_write_retries_total_( + "ha_oplog_etcd_write_retries_total", + "Total number of etcd write retry attempts"), + oplog_watch_disconnections_total_( + "ha_oplog_watch_disconnections_total", + "Total number of OpLog watch disconnections"), + oplog_applied_entries_total_( + "ha_oplog_applied_entries_total", + "Total number of OpLog entries successfully applied"), + + // Latency Histograms (buckets in microseconds) + // 100us, 500us, 1ms, 5ms, 10ms, 50ms, 100ms, 500ms, 1s, 5s + oplog_etcd_write_latency_us_( + "ha_oplog_etcd_write_latency_us", + "Latency of etcd write operations in microseconds", + {100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000, 5000000}), + oplog_apply_latency_us_( + "ha_oplog_apply_latency_us", + "Latency of OpLog entry application in microseconds", + {10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000}), + + // State Machine + standby_state_( + "ha_standby_state", + "Current state of the Standby service (0=STOPPED, 1=CONNECTING, " + "2=SYNCING, 3=WATCHING, 4=RECOVERING, 5=RECONNECTING, " + "6=PROMOTING, 7=PROMOTED, 8=FAILED)"), + state_transitions_total_( + "ha_state_transitions_total", + "Total number of Standby state machine transitions") { + // Initialize gauges to 0 for proper Prometheus output + oplog_last_sequence_id_.update(0); + oplog_applied_sequence_id_.update(0); + oplog_standby_lag_.update(0); + oplog_pending_entries_.update(0); + pending_mutation_queue_size_.update(0); + standby_state_.update(0); +} + +// ========== OpLog Sequence Metrics (Gauge) ========== + +void HAMetricManager::set_oplog_last_sequence_id(int64_t seq_id) { + oplog_last_sequence_id_.update(seq_id); +} + +int64_t HAMetricManager::get_oplog_last_sequence_id() { + return static_cast(oplog_last_sequence_id_.value()); +} + +void HAMetricManager::set_oplog_applied_sequence_id(int64_t seq_id) { + oplog_applied_sequence_id_.update(seq_id); +} + +int64_t HAMetricManager::get_oplog_applied_sequence_id() { + return static_cast(oplog_applied_sequence_id_.value()); +} + +void HAMetricManager::set_oplog_standby_lag(int64_t lag) { + oplog_standby_lag_.update(lag); +} + +int64_t HAMetricManager::get_oplog_standby_lag() { + return static_cast(oplog_standby_lag_.value()); +} + +void HAMetricManager::set_oplog_pending_entries(int64_t count) { + oplog_pending_entries_.update(count); +} + +int64_t HAMetricManager::get_oplog_pending_entries() { + return static_cast(oplog_pending_entries_.value()); +} + +void HAMetricManager::set_pending_mutation_queue_size(int64_t size) { + pending_mutation_queue_size_.update(size); +} + +int64_t HAMetricManager::get_pending_mutation_queue_size() { + return static_cast(pending_mutation_queue_size_.value()); +} + +// ========== Error Counters ========== + +void HAMetricManager::inc_oplog_skipped_entries(int64_t val) { + oplog_skipped_entries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_skipped_entries_total() { + return static_cast(oplog_skipped_entries_total_.value()); +} + +void HAMetricManager::inc_oplog_checksum_failures(int64_t val) { + oplog_checksum_failures_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_checksum_failures_total() { + return static_cast(oplog_checksum_failures_total_.value()); +} + +void HAMetricManager::inc_oplog_gap_resolve_attempts(int64_t val) { + oplog_gap_resolve_attempts_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_gap_resolve_attempts_total() { + return static_cast(oplog_gap_resolve_attempts_total_.value()); +} + +void HAMetricManager::inc_oplog_gap_resolve_success(int64_t val) { + oplog_gap_resolve_success_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_gap_resolve_success_total() { + return static_cast(oplog_gap_resolve_success_total_.value()); +} + +void HAMetricManager::inc_oplog_etcd_write_failures(int64_t val) { + oplog_etcd_write_failures_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_etcd_write_failures_total() { + return static_cast(oplog_etcd_write_failures_total_.value()); +} + +void HAMetricManager::inc_oplog_etcd_write_retries(int64_t val) { + oplog_etcd_write_retries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_etcd_write_retries_total() { + return static_cast(oplog_etcd_write_retries_total_.value()); +} + +void HAMetricManager::inc_oplog_watch_disconnections(int64_t val) { + oplog_watch_disconnections_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_watch_disconnections_total() { + return static_cast(oplog_watch_disconnections_total_.value()); +} + +void HAMetricManager::inc_oplog_applied_entries(int64_t val) { + oplog_applied_entries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_applied_entries_total() { + return static_cast(oplog_applied_entries_total_.value()); +} + +// ========== Latency Histograms ========== + +void HAMetricManager::observe_oplog_etcd_write_latency_us(int64_t latency_us) { + oplog_etcd_write_latency_us_.observe(latency_us); +} + +void HAMetricManager::observe_oplog_apply_latency_us(int64_t latency_us) { + oplog_apply_latency_us_.observe(latency_us); +} + +// ========== State Machine Metrics ========== + +void HAMetricManager::set_standby_state(int64_t state_value) { + standby_state_.update(state_value); +} + +int64_t HAMetricManager::get_standby_state() { + return static_cast(standby_state_.value()); +} + +void HAMetricManager::inc_state_transitions(int64_t val) { + state_transitions_total_.inc(val); +} + +int64_t HAMetricManager::get_state_transitions_total() { + return static_cast(state_transitions_total_.value()); +} + +// ========== Serialization ========== + +std::string HAMetricManager::serialize_metrics() { + std::stringstream ss; + + // Helper lambda to serialize a metric + auto serialize_metric = [&ss](auto& metric) { + std::string metric_str; + metric.serialize(metric_str); + ss << metric_str; + }; + + // Gauges + serialize_metric(oplog_last_sequence_id_); + serialize_metric(oplog_applied_sequence_id_); + serialize_metric(oplog_standby_lag_); + serialize_metric(oplog_pending_entries_); + serialize_metric(pending_mutation_queue_size_); + serialize_metric(standby_state_); + + // Counters + serialize_metric(oplog_skipped_entries_total_); + serialize_metric(oplog_checksum_failures_total_); + serialize_metric(oplog_gap_resolve_attempts_total_); + serialize_metric(oplog_gap_resolve_success_total_); + serialize_metric(oplog_etcd_write_failures_total_); + serialize_metric(oplog_etcd_write_retries_total_); + serialize_metric(oplog_watch_disconnections_total_); + serialize_metric(oplog_applied_entries_total_); + serialize_metric(state_transitions_total_); + + // Histograms + serialize_metric(oplog_etcd_write_latency_us_); + serialize_metric(oplog_apply_latency_us_); + + return ss.str(); +} + +std::string HAMetricManager::get_summary_string() { + std::stringstream ss; + ss << "HA Metrics Summary: "; + ss << "last_seq=" << get_oplog_last_sequence_id(); + ss << ", applied_seq=" << get_oplog_applied_sequence_id(); + ss << ", lag=" << get_oplog_standby_lag(); + ss << ", pending=" << get_oplog_pending_entries(); + ss << ", mutation_queue=" << get_pending_mutation_queue_size(); + ss << ", skipped=" << get_oplog_skipped_entries_total(); + ss << ", checksum_fail=" << get_oplog_checksum_failures_total(); + ss << ", etcd_fail=" << get_oplog_etcd_write_failures_total(); + ss << ", watch_disconn=" << get_oplog_watch_disconnections_total(); + ss << ", state=" << get_standby_state(); + return ss.str(); +} + +} // namespace mooncake + diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp new file mode 100644 index 0000000000..a843e63d95 --- /dev/null +++ b/mooncake-store/src/hot_standby_service.cpp @@ -0,0 +1,589 @@ +#include "hot_standby_service.h" + +#include + +#include +#include + +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "ha_metric_manager.h" +#include "master_service.h" +#include "oplog_applier.h" +#include "oplog_manager.h" +#include "oplog_watcher.h" + +namespace mooncake { + +HotStandbyService::HotStandbyService(const HotStandbyConfig& config) + : config_(config) { + metadata_store_ = std::make_unique(); + // OpLogApplier will be re-created in Start() with the resolved cluster_id + // to enable etcd-based operations (e.g. requesting missing OpLog entries). + // Here we construct a minimal instance so that local metadata operations + // are available before etcd wiring is completed. + oplog_applier_ = std::make_unique(metadata_store_.get()); + + // Register callback for state change logging and metrics. + state_machine_.RegisterCallback([](StandbyState old_state, StandbyState new_state, StandbyEvent event) { + LOG(INFO) << "HotStandbyService state changed: " + << StandbyStateToString(old_state) << " -> " + << StandbyStateToString(new_state) + << " (event: " << StandbyEventToString(event) << ")"; + + // Update HA metrics + HAMetricManager::instance().set_standby_state(static_cast(new_state)); + HAMetricManager::instance().inc_state_transitions(); + + // Track watch disconnections + if (event == StandbyEvent::WATCH_BROKEN || event == StandbyEvent::DISCONNECTED) { + HAMetricManager::instance().inc_oplog_watch_disconnections(); + } + }); +} + +// StandbyMetadataStore implementation +bool HotStandbyService::StandbyMetadataStore::PutMetadata( + const std::string& key, const StandbyObjectMetadata& metadata) { + std::lock_guard lock(mutex_); + store_[key] = metadata; + VLOG(2) << "StandbyMetadataStore: stored metadata for key=" << key + << ", replicas=" << metadata.replicas.size() + << ", size=" << metadata.size; + return true; +} + +bool HotStandbyService::StandbyMetadataStore::Put(const std::string& key, + const std::string& payload) { + // Legacy interface - create empty metadata + StandbyObjectMetadata metadata; + std::lock_guard lock(mutex_); + store_[key] = metadata; + return true; +} + +const StandbyObjectMetadata* HotStandbyService::StandbyMetadataStore::GetMetadata( + const std::string& key) const { + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it != store_.end()) { + return &it->second; + } + return nullptr; +} + +bool HotStandbyService::StandbyMetadataStore::Remove(const std::string& key) { + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it != store_.end()) { + store_.erase(it); + return true; + } + return false; +} + +bool HotStandbyService::StandbyMetadataStore::Exists( + const std::string& key) const { + std::lock_guard lock(mutex_); + return store_.find(key) != store_.end(); +} + +size_t HotStandbyService::StandbyMetadataStore::GetKeyCount() const { + std::lock_guard lock(mutex_); + return store_.size(); +} + +void HotStandbyService::StandbyMetadataStore::Snapshot( + std::vector>& out) const { + std::lock_guard lock(mutex_); + out.clear(); + out.reserve(store_.size()); + for (const auto& kv : store_) { + out.emplace_back(kv.first, kv.second); + } +} + +HotStandbyService::~HotStandbyService() { + Stop(); +} + +ErrorCode HotStandbyService::Start(const std::string& primary_address, + const std::string& etcd_endpoints, + const std::string& cluster_id) { + std::lock_guard lock(mutex_); + + // Use state machine to check if already running + if (IsRunning()) { + LOG(WARNING) << "HotStandbyService is already running"; + return ErrorCode::OK; + } + + // Trigger START event + auto result = state_machine_.ProcessEvent(StandbyEvent::START); + if (!result.allowed) { + LOG(ERROR) << "Cannot start HotStandbyService: " << result.reason; + return ErrorCode::INTERNAL_ERROR; // State machine rejected START + } + + config_.primary_address = primary_address; + etcd_endpoints_ = etcd_endpoints; + cluster_id_ = cluster_id; + +#ifdef STORE_USE_ETCD + // Connect to etcd + ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(etcd_endpoints.c_str()); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to connect to etcd: " << etcd_endpoints; + state_machine_.ProcessEvent(StandbyEvent::CONNECTION_FAILED); + return err; + } + + // Transition to SYNCING state + state_machine_.ProcessEvent(StandbyEvent::CONNECTED); + + // Preserve existing local state if HotStandbyService is restarted in-process: + // - metadata_store_ may already contain real-time metadata + // - oplog_applier_ may already have expected_sequence_id_ + uint64_t local_last_seq_id = 0; + if (oplog_applier_) { + uint64_t expected = oplog_applier_->GetExpectedSequenceId(); + local_last_seq_id = expected > 0 ? expected - 1 : 0; + } + const bool has_local_metadata = + metadata_store_ && metadata_store_->GetKeyCount() > 0; + const bool has_local_state = has_local_metadata && local_last_seq_id > 0; + + // Recreate OpLogApplier with cluster_id (for requesting missing OpLog). + // If we had local state, recover to keep sequence continuity. + oplog_applier_ = std::make_unique(metadata_store_.get(), cluster_id); + if (has_local_state) { + LOG(INFO) << "Standby warm start: reuse local metadata (keys=" + << metadata_store_->GetKeyCount() + << "), recover last_seq_id=" << local_last_seq_id; + oplog_applier_->Recover(local_last_seq_id); + } + + // Create OpLogWatcher with state machine callback + oplog_watcher_ = std::make_unique( + etcd_endpoints, cluster_id, oplog_applier_.get()); + + // Register callback for watcher events + oplog_watcher_->SetStateCallback([this](StandbyEvent event) { + OnWatcherEvent(event); + }); + + // Bootstrap: + // - If we already have local state (warm start), do NOT reload snapshot. + // - Otherwise (cold start/new standby), try snapshot (if enabled) then replay OpLog. + uint64_t baseline_seq_id = has_local_state ? local_last_seq_id : 0; + if (!has_local_state && config_.enable_snapshot_bootstrap && snapshot_provider_) { + std::string snapshot_id; + uint64_t snapshot_seq_id = 0; + std::vector> snapshot; + if (snapshot_provider_->LoadLatestSnapshot(cluster_id_, snapshot_id, snapshot_seq_id, + snapshot)) { + LOG(INFO) << "Loaded snapshot: snapshot_id=" << snapshot_id + << ", snapshot_seq_id=" << snapshot_seq_id + << ", keys=" << snapshot.size(); + // Apply snapshot into local standby store. + for (const auto& kv : snapshot) { + metadata_store_->PutMetadata(kv.first, kv.second); + } + // Align applier to snapshot boundary. + oplog_applier_->Recover(snapshot_seq_id); + baseline_seq_id = snapshot_seq_id; + } else { + LOG(INFO) << "No snapshot available (or provider not ready), falling back to OpLog-only bootstrap"; + } + } + + // Read historical OpLog entries since baseline_seq_id. + uint64_t last_applied_seq_id = baseline_seq_id; + + // Start OpLogWatcher with a consistent "read then watch(from revision+1)" sequence. + if (!oplog_watcher_->StartFromSequenceId(last_applied_seq_id)) { + LOG(WARNING) << "Failed to start OpLogWatcher from sequence_id=" + << last_applied_seq_id << ", continuing anyway"; + state_machine_.ProcessEvent(StandbyEvent::SYNC_FAILED); + } else { + // Transition to WATCHING state after successful sync + state_machine_.ProcessEvent(StandbyEvent::SYNC_COMPLETE); + } + + // Start background threads + replication_thread_ = std::thread(&HotStandbyService::ReplicationLoop, this); + if (config_.enable_verification) { + verification_thread_ = + std::thread(&HotStandbyService::VerificationLoop, this); + } + + LOG(INFO) << "HotStandbyService started, watching etcd OpLog for cluster: " + << cluster_id << ", state=" << StandbyStateToString(GetState()); + return ErrorCode::OK; +#else + state_machine_.ProcessEvent(StandbyEvent::FATAL_ERROR); + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot start HotStandbyService"; + return ErrorCode::INTERNAL_ERROR; +#endif +} + +void HotStandbyService::OnWatcherEvent(StandbyEvent event) { + state_machine_.ProcessEvent(event); +} + +void HotStandbyService::Stop() { + if (!IsRunning() && GetState() != StandbyState::PROMOTING) { + return; + } + + // Trigger STOP event + state_machine_.ProcessEvent(StandbyEvent::STOP); + + // Stop OpLogWatcher + if (oplog_watcher_) { + oplog_watcher_->Stop(); + oplog_watcher_.reset(); + } + + // Wait for threads to finish + if (replication_thread_.joinable()) { + replication_thread_.join(); + } + if (verification_thread_.joinable()) { + verification_thread_.join(); + } + + LOG(INFO) << "HotStandbyService stopped, final_state=" << StandbyStateToString(GetState()); +} + +StandbySyncStatus HotStandbyService::GetSyncStatus() const { + StandbySyncStatus status; + + // Get applied sequence ID from OpLogApplier + if (oplog_applier_) { + status.applied_seq_id = oplog_applier_->GetExpectedSequenceId() - 1; + if (status.applied_seq_id == 0) { + status.applied_seq_id = applied_seq_id_.load(); // Fallback + } + } else { + status.applied_seq_id = applied_seq_id_.load(); + } + + // Primary sequence ID (best-effort): updated by ReplicationLoop via etcd `/latest`. + status.primary_seq_id = primary_seq_id_.load(); + + // Use state machine for connection status + status.is_connected = IsConnected(); + status.state = GetState(); + status.time_in_state = state_machine_.GetTimeInCurrentState(); + + if (status.primary_seq_id > status.applied_seq_id) { + status.lag_entries = status.primary_seq_id - status.applied_seq_id; + } else { + status.lag_entries = 0; + } + + // Lag time is currently reported as 0; if needed we can extend the + // protocol to propagate primary timestamps and compute a real value. + status.lag_time = std::chrono::milliseconds(0); + status.is_syncing = IsRunning() && IsConnected(); + + return status; +} + +bool HotStandbyService::IsReadyForPromotion() const { + // Use state machine to check if ready for promotion + if (!state_machine_.IsReadyForPromotion()) { + return false; + } + + StandbySyncStatus status = GetSyncStatus(); + + // Allow promotion even with large lag - the new Primary can continue + // syncing remaining OpLog entries from etcd after promotion. + // Log a warning if lag is large, but don't block promotion. + if (status.lag_entries > config_.max_replication_lag_entries) { + LOG(WARNING) << "Standby has large replication lag: " << status.lag_entries + << " entries (threshold: " << config_.max_replication_lag_entries + << "). Promotion will proceed, but remaining OpLog entries " + << "will be synced after promotion."; + } + + return true; +} + +std::unique_ptr HotStandbyService::Promote() { + std::lock_guard lock(mutex_); + + if (!IsReadyForPromotion()) { + LOG(ERROR) << "Standby is not ready for promotion, state=" + << StandbyStateToString(GetState()); + return nullptr; + } + + // Trigger PROMOTE event + auto result = state_machine_.ProcessEvent(StandbyEvent::PROMOTE); + if (!result.allowed) { + LOG(ERROR) << "Cannot promote: " << result.reason; + return nullptr; + } + + StandbySyncStatus status = GetSyncStatus(); + uint64_t current_applied_seq_id = status.applied_seq_id; + + LOG(INFO) << "Promoting Standby to Primary. Applied seq_id: " + << current_applied_seq_id + << ", lag: " << status.lag_entries << " entries" + << ", state: " << StandbyStateToString(GetState()); + + // Final catch-up sync before promotion. + // IMPORTANT: + // - Do NOT rely on `lag_entries` here because primary_seq_id_ is best-effort. + // - Stop OpLogWatcher first to avoid concurrent Apply from watch callbacks. + if (oplog_watcher_) { + oplog_watcher_->Stop(); + } + + // Best-effort: resolve any outstanding gaps with retry before promotion. + // Do NOT block promotion if gaps cannot be fetched after max retries. + static constexpr int kMaxGapResolveRetries = 3; + if (oplog_applier_) { + for (int retry = 0; retry < kMaxGapResolveRetries; ++retry) { + auto res = oplog_applier_->TryResolveGapsOnceForPromotion(/*max_ids=*/1024); + if (res.attempted == 0) { + // No gaps to resolve + break; + } + LOG(INFO) << "Promotion gap resolve (attempt " << (retry + 1) << "/" + << kMaxGapResolveRetries << "): attempted=" << res.attempted + << ", fetched=" << res.fetched + << ", applied_deletes=" << res.applied_deletes; + if (res.fetched == res.attempted) { + // All gaps resolved successfully + break; + } + // Some gaps failed, retry after short delay + if (retry + 1 < kMaxGapResolveRetries) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } + } + + LOG(INFO) << "Final catch-up sync from etcd before promotion..."; + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + const size_t batch_size = 1000; + + // P0 fix: Prevent underflow when current_applied_seq_id is 0 + // ReadOpLogSince reads entries with seq > given_seq, so we pass current_applied_seq_id directly + uint64_t read_from_seq = current_applied_seq_id; // Will read entries with seq > read_from_seq + + // P1 fix: Add timeout control to prevent infinite blocking + static constexpr size_t kMaxCatchUpBatches = 100; // Max 100 batches * 1000 = 100k entries + static constexpr auto kMaxCatchUpDuration = std::chrono::seconds(30); + auto catch_up_start = std::chrono::steady_clock::now(); + + size_t total_applied = 0; + size_t batch_count = 0; + + for (;;) { + // Check timeout + auto elapsed = std::chrono::steady_clock::now() - catch_up_start; + if (elapsed > kMaxCatchUpDuration) { + LOG(WARNING) << "Final catch-up: timeout after " + << std::chrono::duration_cast(elapsed).count() + << "s. Proceeding with promotion. total_applied=" << total_applied; + break; + } + + // Check batch limit + if (batch_count >= kMaxCatchUpBatches) { + LOG(WARNING) << "Final catch-up: reached max batch limit (" << kMaxCatchUpBatches + << "). Proceeding with promotion. total_applied=" << total_applied; + break; + } + + std::vector batch; + ErrorCode read_err = oplog_store.ReadOpLogSince(read_from_seq, batch_size, batch); + if (read_err != ErrorCode::OK) { + LOG(WARNING) << "Final catch-up: failed to read OpLog since seq=" + << read_from_seq << ", err=" << static_cast(read_err) + << ". Proceeding with promotion."; + break; + } + if (batch.empty()) { + break; + } + size_t applied = oplog_applier_->ApplyOpLogEntries(batch); + total_applied += applied; + read_from_seq = batch.back().sequence_id; // Next read will get entries > this seq + ++batch_count; + } + LOG(INFO) << "Final catch-up sync done. total_applied=" << total_applied + << ", batches=" << batch_count; + + // Transition to PROMOTED state + state_machine_.ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + + // Stop replication (OpLogWatcher will stop watching). + // Note: This will trigger STOP event, transitioning to STOPPED. + Stop(); + + // Design note: MasterService creation and initialization are handled by + // MasterServiceSupervisor::Start() after leader election. The + // responsibility of HotStandbyService::Promote() is limited to ensuring + // that all remaining OpLog entries are applied before the new Primary + // starts serving requests. + + LOG(INFO) << "Standby promoted to Primary successfully. " + << "All remaining OpLog entries have been synced."; + + // Return nullptr - actual MasterService creation happens externally + // The caller (MasterServiceSupervisor) will create the MasterService + // with the appropriate configuration. + return nullptr; +} + +size_t HotStandbyService::GetMetadataCount() const { + std::lock_guard lock(mutex_); + return metadata_store_ ? metadata_store_->GetKeyCount() : 0; +} + +uint64_t HotStandbyService::GetLatestAppliedSequenceId() const { + std::lock_guard lock(mutex_); + if (oplog_applier_) { + uint64_t expected_seq = oplog_applier_->GetExpectedSequenceId(); + // GetExpectedSequenceId returns the next expected sequence_id, + // so the latest applied is expected_seq - 1 + return expected_seq > 0 ? expected_seq - 1 : 0; + } + return applied_seq_id_.load(); +} + +bool HotStandbyService::ExportMetadataSnapshot( + std::vector>& out) const { + std::lock_guard lock(mutex_); + if (!metadata_store_) { + out.clear(); + return false; + } + metadata_store_->Snapshot(out); + return true; +} + +void HotStandbyService::SetSnapshotProvider(std::unique_ptr provider) { + std::lock_guard lock(mutex_); + if (provider) { + snapshot_provider_ = std::move(provider); + } else { + snapshot_provider_ = std::make_unique(); + } +} + +void HotStandbyService::ReplicationLoop() { + LOG(INFO) << "Replication loop started (etcd-based OpLog sync)"; + + // With etcd-based OpLog sync, OpLogWatcher handles the actual watching + // in its own thread. This loop now just monitors the status and updates + // metrics. + + while (IsRunning()) { + if (!IsConnected()) { + // Not connected - wait a bit before checking again + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + // Update applied_seq_id from OpLogApplier + if (oplog_applier_) { + uint64_t current_applied = oplog_applier_->GetExpectedSequenceId() - 1; + if (current_applied > 0) { + applied_seq_id_.store(current_applied); + } + } + + // Update primary_seq_id by querying etcd `/latest` (best-effort). + // Note: `/latest` is batch-updated on Primary, so this is for monitoring only. +#ifdef STORE_USE_ETCD + if (!cluster_id_.empty()) { + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + uint64_t latest_seq = 0; + ErrorCode err = oplog_store.GetLatestSequenceId(latest_seq); + if (err == ErrorCode::OK) { + primary_seq_id_.store(latest_seq); + } + } +#endif + + // Sleep and check again + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + + LOG(INFO) << "Replication loop stopped"; +} + +void HotStandbyService::VerificationLoop() { + LOG(INFO) << "Verification loop started"; + + while (IsRunning()) { + std::this_thread::sleep_for( + std::chrono::seconds(config_.verification_interval_sec)); + + if (!IsConnected()) { + continue; + } + + // Verification is not yet implemented. When enabled, this loop is + // expected to: + // 1) sample keys from the local metadata store, + // 2) calculate checksums, + // 3) send a verification request to the Primary, and + // 4) handle any mismatches that are detected. + VLOG(1) << "Verification check skipped (feature not implemented), state=" + << StandbyStateToString(GetState()); + } + + LOG(INFO) << "Verification loop stopped"; +} + +void HotStandbyService::ApplyOpLogEntry(const OpLogEntry& entry) { + // NOTE: This method is deprecated. OpLog entries are now applied via + // OpLogApplier, which is called by OpLogWatcher. This method is kept + // for backward compatibility but should not be used in the new etcd-based + // implementation. + + // Update applied_seq_id for status tracking + applied_seq_id_.store(entry.sequence_id); + + // The actual application is handled by OpLogApplier via OpLogWatcher + VLOG(2) << "ApplyOpLogEntry called (deprecated), sequence_id=" + << entry.sequence_id << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; +} + +void HotStandbyService::ProcessOpLogBatch( + const std::vector& entries) { + for (const auto& entry : entries) { + ApplyOpLogEntry(entry); + } +} + +bool HotStandbyService::ConnectToPrimary() { + // With etcd-based OpLog sync, connection is handled by OpLogWatcher + // This method is kept for compatibility but is no longer used + LOG(INFO) << "ConnectToPrimary called (no-op with etcd-based sync)"; + return true; +} + +void HotStandbyService::DisconnectFromPrimary() { + // With etcd-based OpLog sync, disconnection is handled by OpLogWatcher + // This method is kept for compatibility + if (IsConnected()) { + state_machine_.ProcessEvent(StandbyEvent::DISCONNECTED); + replication_stream_.reset(); + LOG(INFO) << "Disconnected from Primary (etcd-based sync), state=" + << StandbyStateToString(GetState()); + } +} + +} // namespace mooncake + diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index e049cb10eb..7b4a0450aa 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -2,19 +2,138 @@ #include #include -#include #include #include +#include #include +#include +#include "allocator.h" +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "ha_metric_manager.h" #include "master_metric_manager.h" +#include "metadata_store.h" // For MetadataPayload #include "segment.h" #include "types.h" +// replication_service.h removed - using etcd-based OpLog sync instead namespace mooncake { +namespace { + +// A minimal allocator implementation used only to keep AllocatedBuffer handles +// "valid" after standby promotion. It does NOT own memory. +class DummyBufferAllocator final : public BufferAllocatorBase { + public: + DummyBufferAllocator(std::string segment_name, std::string transport_endpoint) + : segment_name_(std::move(segment_name)), + transport_endpoint_(std::move(transport_endpoint)) {} + + std::unique_ptr allocate(size_t /*size*/) override { + return nullptr; + } + void deallocate(AllocatedBuffer* /*handle*/) override { + // no-op: we don't own memory + } + size_t capacity() const override { return kAllocatorUnknownFreeSpace; } + size_t size() const override { return 0; } + std::string getSegmentName() const override { return segment_name_; } + std::string getTransportEndpoint() const override { return transport_endpoint_; } + size_t getLargestFreeRegion() const override { return kAllocatorUnknownFreeSpace; } + + private: + std::string segment_name_; + std::string transport_endpoint_; +}; + +static Replica ReplicaFromDescriptor( + const Replica::Descriptor& desc, + const std::shared_ptr& allocator_keepalive) { + if (desc.is_memory_replica()) { + const auto& mem = desc.get_memory_descriptor(); + const auto& bd = mem.buffer_descriptor; + if (!allocator_keepalive) { + // This would make the buffer handle invalid immediately (allocator stored + // as weak_ptr in AllocatedBuffer). Callers restoring from standby should + // always provide a keepalive allocator. + LOG(ERROR) << "ReplicaFromDescriptor(memory) missing keepalive allocator, " + << "transport_endpoint=" << bd.transport_endpoint_; + } + + auto buf = std::make_unique( + allocator_keepalive, reinterpret_cast(bd.buffer_address_), + static_cast(bd.size_)); + return Replica(std::move(buf), desc.status); + } + if (desc.is_disk_replica()) { + const auto& disk = desc.get_disk_descriptor(); + return Replica(disk.file_path, disk.object_size, desc.status); + } + const auto& ld = desc.get_local_disk_descriptor(); + UUID client_id{ld.client_id_first, ld.client_id_second}; + return Replica(client_id, ld.object_size, ld.transport_endpoint, desc.status); +} + +} // namespace + MasterService::MasterService() : MasterService(MasterServiceConfig()) {} +std::string MasterService::SerializeMetadataForOpLog(const ObjectMetadata& metadata) const { + MetadataPayload payload; + payload.client_id_first = metadata.client_id.first; + payload.client_id_second = metadata.client_id.second; + payload.size = metadata.size; + + // Extract replica descriptors + payload.replicas.reserve(metadata.replicas.size()); + for (const auto& replica : metadata.replicas) { + payload.replicas.push_back(replica.get_descriptor()); + } + + // NOTE: Lease information is NOT serialized because: + // 1. Standby does not perform eviction, so lease info is not used + // 2. After promotion, new Primary should grant fresh leases, not restore old ones + + // Serialize to JSON + std::string json_str; + struct_json::to_json(payload, json_str); + return json_str; +} + +std::string MasterService::SerializeMetadataForOpLogWithoutMemReplicas( + const ObjectMetadata& metadata) const { + MetadataPayload payload; + payload.client_id_first = metadata.client_id.first; + payload.client_id_second = metadata.client_id.second; + payload.size = metadata.size; + + payload.replicas.reserve(metadata.replicas.size()); + for (const auto& replica : metadata.replicas) { + if (replica.type() == ReplicaType::MEMORY) { + continue; + } + payload.replicas.push_back(replica.get_descriptor()); + } + + std::string json_str; + struct_json::to_json(payload, json_str); + return json_str; +} + +std::string MasterService::SerializeMetadataForOpLogFromReplicaDescriptors( + const UUID& client_id, uint64_t size, + const std::vector& replicas) const { + MetadataPayload payload; + payload.client_id_first = client_id.first; + payload.client_id_second = client_id.second; + payload.size = size; + payload.replicas = replicas; + std::string json_str; + struct_json::to_json(payload, json_str); + return json_str; +} + MasterService::MasterService(const MasterServiceConfig& config) : default_kv_lease_ttl_(config.default_kv_lease_ttl), default_kv_soft_pin_ttl_(config.default_kv_soft_pin_ttl), @@ -72,6 +191,168 @@ MasterService::MasterService(const MasterServiceConfig& config) MasterMetricManager::instance().inc_total_file_capacity( global_file_segment_size_); } + + // Initialize EtcdOpLogStore if HA is enabled + // Note: This requires STORE_USE_ETCD to be enabled at compile time + // Note: etcd connection should be established before MasterService construction + // (e.g., in MasterServiceSupervisor), so we can use the existing connection +#ifdef STORE_USE_ETCD + if (enable_ha_ && !cluster_id_.empty()) { + // Try to create EtcdOpLogStore - if etcd is not connected, operations will fail + // but we can still use memory buffer as fallback + // Writer: enable batch update for `/latest` to reduce etcd write pressure. + auto etcd_oplog_store = + std::make_shared(cluster_id_, /*enable_latest_seq_batch_update=*/true); + oplog_manager_.SetEtcdOpLogStore(etcd_oplog_store); + // Fence against restart/promotion regressions: initialize OpLogManager + // to the maximum existing sequence_id in etcd so we don't collide/overwrite. + uint64_t max_seq = 0; + if (etcd_oplog_store->GetMaxSequenceId(max_seq) == ErrorCode::OK) { + oplog_manager_.SetInitialSequenceId(max_seq); + } + LOG(INFO) << "EtcdOpLogStore initialized for cluster_id=" + << cluster_id_ << " (etcd connection should be established " + << "before MasterService construction)"; + } else if (enable_ha_) { + LOG(WARNING) << "HA mode enabled but cluster_id is empty, " + "OpLog will only be stored in memory buffer"; + } +#else + if (enable_ha_) { + LOG(WARNING) << "HA mode enabled but STORE_USE_ETCD is not enabled at " + "compile time, OpLog will only be stored in memory buffer. " + "Recompile with -DSTORE_USE_ETCD=ON to enable etcd support."; + } +#endif + + // Start pending durable mutation retry thread (HA only). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + pending_mutations_running_.store(true); + pending_mutations_thread_ = + std::thread(&MasterService::PendingMutationWorker, this); + } +#endif +} + +// Helper function to append an OpLog entry. +// In the current etcd-based design: +// - OpLogManager always appends to its in-memory buffer +// - If EtcdOpLogStore is configured (HA mode), OpLogManager also writes to etcd +// synchronously (best-effort; see OpLogManager::Append). +void MasterService::AppendOpLogAndNotify(OpType type, const std::string& key, + const std::string& payload) { + oplog_manager_.Append(type, key, payload); +} + +auto MasterService::AppendOpLogAndNotifyDurable(OpType type, const std::string& key, + const std::string& payload) + -> tl::expected { +#ifdef STORE_USE_ETCD + // In HA mode, EtcdOpLogStore should have been configured into OpLogManager. + // For safety, treat missing store as an error for durable ops. + // Best-effort synchronous retries to absorb transient etcd blips. + // + // IMPORTANT: + // sequence_id must be allocated ONCE (pre-allocation) and retried with the same + // OpLogEntry, otherwise multiple attempts would allocate multiple sequence_ids + // for a single logical operation. + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, payload); + ErrorCode err = PersistOpLogEntryWithSyncRetries(entry); + if (err == ErrorCode::OK) { + return entry.sequence_id; + } + return tl::make_unexpected(err); +#else + (void)type; + (void)key; + (void)payload; + return tl::make_unexpected(ErrorCode::ETCD_OPERATION_ERROR); +#endif +} + +void MasterService::RestoreFromStandbySnapshot( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id) { + // 1) Ensure OpLog sequence continues without regression after failover. + // Prefer reading the true max seq from etcd (stronger than standby_last_seq), + // fall back to caller-provided initial_oplog_sequence_id. + uint64_t start_seq = initial_oplog_sequence_id; +#ifdef STORE_USE_ETCD + if (enable_ha_ && !cluster_id_.empty()) { + EtcdOpLogStore store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + uint64_t max_seq = 0; + if (store.GetMaxSequenceId(max_seq) == ErrorCode::OK) { + start_seq = std::max(start_seq, max_seq); + } + } +#endif + oplog_manager_.SetInitialSequenceId(start_seq); + + // 2) Restore metadata entries. + // Keep dummy allocators alive for restored memory replicas. AllocatedBuffer + // only holds a weak_ptr to allocator, so without this keepalive map the + // allocator would expire immediately and transport_endpoint_ would be lost. + standby_allocator_keepalive_.clear(); + auto get_keepalive_allocator = + [this](const std::string& transport_endpoint) + -> std::shared_ptr { + auto it = standby_allocator_keepalive_.find(transport_endpoint); + if (it != standby_allocator_keepalive_.end()) { + return it->second; + } + auto alloc = std::make_shared( + /*segment_name=*/std::string(), transport_endpoint); + standby_allocator_keepalive_.emplace(transport_endpoint, alloc); + return alloc; + }; + + const auto now = std::chrono::steady_clock::now(); + size_t restored = 0; + for (const auto& kv : snapshot) { + const std::string& key = kv.first; + const StandbyObjectMetadata& sm = kv.second; + + std::vector replicas; + replicas.reserve(sm.replicas.size()); + for (const auto& rd : sm.replicas) { + if (rd.is_memory_replica()) { + const auto& bd = rd.get_memory_descriptor().buffer_descriptor; + replicas.emplace_back( + ReplicaFromDescriptor(rd, get_keepalive_allocator(bd.transport_endpoint_))); + } else { + replicas.emplace_back(ReplicaFromDescriptor(rd, nullptr)); + } + } + + // NOTE: Lease information is NOT restored because: + // 1. Standby does not use lease info (no eviction) + // 2. New Primary should grant fresh leases after promotion + // 3. Restoring old lease TTLs could cause immediate eviction if they're expired + const bool enable_soft_pin = false; // Will be set by new Primary if needed + + const size_t shard_idx = getShardIndex(key); + MutexLocker lock(&metadata_shards_[shard_idx].mutex); + + // Overwrite existing key if any. + metadata_shards_[shard_idx].metadata.erase(key); + auto [it, inserted] = metadata_shards_[shard_idx].metadata.emplace( + std::piecewise_construct, std::forward_as_tuple(key), + std::forward_as_tuple(sm.client_id, now, static_cast(sm.size), + std::move(replicas), enable_soft_pin)); + (void)inserted; + + // Lease will be granted by new Primary when objects are accessed + // (via GetReplicaList, ExistKey, etc.) + + // Objects restored from PUT_END are expected to be completed. + metadata_shards_[shard_idx].processing_keys.erase(key); + + restored++; + } + + LOG(INFO) << "Restored metadata from standby snapshot: restored_keys=" + << restored << ", initial_oplog_sequence_id=" << initial_oplog_sequence_id; } MasterService::~MasterService() { @@ -84,6 +365,237 @@ MasterService::~MasterService() { if (client_monitor_thread_.joinable()) { client_monitor_thread_.join(); } + +#ifdef STORE_USE_ETCD + if (pending_mutations_running_.load()) { + pending_mutations_running_.store(false); + pending_mutations_cv_.notify_all(); + if (pending_mutations_thread_.joinable()) { + pending_mutations_thread_.join(); + } + } +#endif +} + +void MasterService::EnqueuePendingMutation(PendingMutation m) { + m.attempt = 0; + m.next_retry_at = std::chrono::steady_clock::now(); + size_t queue_size = 0; + { + std::lock_guard lg(pending_mutations_mutex_); + if (pending_mutations_.size() >= kMaxPendingMutations) { + // Queue full: drop oldest mutation to prevent unbounded growth. + // Log warning for monitoring. + LOG(WARNING) << "PendingMutation queue full (size=" << pending_mutations_.size() + << "), dropping oldest mutation. key=" << pending_mutations_.front().key + << ", seq=" << pending_mutations_.front().oplog_entry.sequence_id; + pending_mutations_.pop_front(); + } + pending_mutations_.push_back(std::move(m)); + queue_size = pending_mutations_.size(); + } + HAMetricManager::instance().set_pending_mutation_queue_size(static_cast(queue_size)); + pending_mutations_cv_.notify_one(); +} + +ErrorCode MasterService::PersistOpLogEntryWithSyncRetries( + const OpLogEntry& entry) const { +#ifdef STORE_USE_ETCD + static constexpr int kSyncRetries = 3; + static constexpr int kBaseBackoffMs = 20; + ErrorCode persist_err = ErrorCode::ETCD_OPERATION_ERROR; + + auto start_time = std::chrono::steady_clock::now(); + + for (int attempt = 0; attempt < kSyncRetries; ++attempt) { + persist_err = oplog_manager_.PersistEntryToEtcd(entry); + if (persist_err == ErrorCode::OK) { + break; + } + if (attempt > 0) { + HAMetricManager::instance().inc_oplog_etcd_write_retries(); + } + std::this_thread::sleep_for( + std::chrono::milliseconds(kBaseBackoffMs * (1 << attempt))); + } + + auto end_time = std::chrono::steady_clock::now(); + auto latency_us = std::chrono::duration_cast( + end_time - start_time).count(); + HAMetricManager::instance().observe_oplog_etcd_write_latency_us(latency_us); + + if (persist_err == ErrorCode::OK) { + HAMetricManager::instance().set_oplog_last_sequence_id( + static_cast(entry.sequence_id)); + } else { + HAMetricManager::instance().inc_oplog_etcd_write_failures(); + } + + return persist_err; +#else + (void)entry; + return ErrorCode::ETCD_OPERATION_ERROR; +#endif +} + +void MasterService::EnqueueRetryOnPersistFailure( + const char* ctx, const OpLogEntry& entry, ErrorCode persist_err, + PendingMutationKind kind, const std::string& segment_name) { +#ifdef STORE_USE_ETCD + LOG(ERROR) << ctx << ": failed to persist OpLog to etcd, key=" + << entry.object_key << ", seq=" << entry.sequence_id + << ", err=" << persist_err << ". Enqueue retry."; + EnqueuePendingMutation(PendingMutation{ + kind, + entry.object_key, + segment_name, + /*oplog_entry=*/entry}); +#else + (void)ctx; + (void)entry; + (void)persist_err; + (void)kind; + (void)segment_name; +#endif +} + +void MasterService::AppendOrPersistOrEnqueue( + const char* ctx, OpType type, const std::string& key, + const std::string& payload, PendingMutationKind kind, + const std::string& segment_name) { +#ifdef STORE_USE_ETCD + if (enable_ha_) { + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, payload); + ErrorCode persist_err = PersistOpLogEntryWithSyncRetries(entry); + if (persist_err != ErrorCode::OK) { + EnqueueRetryOnPersistFailure(ctx, entry, persist_err, kind, segment_name); + } + } else { + AppendOpLogAndNotify(type, key, payload); + } +#else + // No etcd support at compile time: + // - non-HA: keep best-effort in-memory OpLog for debugging/consistency + // - HA: no-op (constructor already warns) + if (!enable_ha_) { + AppendOpLogAndNotify(type, key, payload); + } + (void)ctx; + (void)kind; + (void)segment_name; +#endif +} + +void MasterService::AppendOrPersistOrEnqueueLazy( + const char* ctx, OpType type, const std::string& key, + const std::function& payload_factory, + PendingMutationKind kind, const std::string& segment_name) { + std::string payload; + bool payload_ready = false; + auto get_payload = [&]() -> const std::string& { + if (!payload_ready) { + payload = payload_factory ? payload_factory() : std::string(); + payload_ready = true; + } + return payload; + }; + +#ifdef STORE_USE_ETCD + if (enable_ha_) { + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, get_payload()); + ErrorCode persist_err = PersistOpLogEntryWithSyncRetries(entry); + if (persist_err != ErrorCode::OK) { + EnqueueRetryOnPersistFailure(ctx, entry, persist_err, kind, segment_name); + } + } else { + AppendOpLogAndNotify(type, key, get_payload()); + } +#else + // No etcd support at compile time: + // - non-HA: keep best-effort in-memory OpLog for debugging/consistency + // - HA: no-op (constructor already warns) + if (!enable_ha_) { + AppendOpLogAndNotify(type, key, get_payload()); + } + (void)ctx; + (void)kind; + (void)segment_name; +#endif +} + +// Return true if processed successfully (done), false if should retry later. +bool MasterService::ProcessPendingMutationOnce(PendingMutation& m) { +#ifndef STORE_USE_ETCD + (void)m; + return true; +#else + const auto now = std::chrono::steady_clock::now(); + if (m.next_retry_at > now) { + return false; + } + + // Retrier responsibility: + // only persist the original pre-allocated OpLogEntry (fixed sequence_id) to etcd. + // Do NOT mutate local metadata here because the caller may have already moved on. + if (m.oplog_entry.sequence_id == 0) { + LOG(WARNING) << "PendingMutation has no pre-allocated OpLogEntry, drop. key=" + << m.key << ", kind=" << static_cast(m.kind); + return true; + } + + ErrorCode err = oplog_manager_.PersistEntryToEtcd(m.oplog_entry); + if (err != ErrorCode::OK) { + return false; + } + return true; +#endif +} + +void MasterService::PendingMutationWorker() { +#ifndef STORE_USE_ETCD + return; +#else + while (pending_mutations_running_.load()) { + PendingMutation m; + bool has_item = false; + { + std::unique_lock lk(pending_mutations_mutex_); + pending_mutations_cv_.wait_for(lk, std::chrono::milliseconds(200), [&] { + return !pending_mutations_running_.load() || !pending_mutations_.empty(); + }); + if (!pending_mutations_running_.load()) { + break; + } + if (pending_mutations_.empty()) { + continue; + } + m = std::move(pending_mutations_.front()); + pending_mutations_.pop_front(); + has_item = true; + } + if (!has_item) { + continue; + } + + const bool done = ProcessPendingMutationOnce(m); + if (done) { + continue; + } + + // Retry with exponential backoff (cap at 30s). + m.attempt++; + const uint32_t exp = std::min(m.attempt, 8); + const auto delay = std::chrono::milliseconds(200u * (1u << exp)); + const auto capped = std::min(delay, std::chrono::milliseconds(30000)); + m.next_retry_at = std::chrono::steady_clock::now() + capped; + + { + std::lock_guard lg(pending_mutations_mutex_); + pending_mutations_.push_back(std::move(m)); + } + pending_mutations_cv_.notify_one(); + } +#endif } auto MasterService::MountSegment(const Segment& segment, const UUID& client_id) @@ -172,10 +684,40 @@ void MasterService::ClearInvalidHandles() { MutexLocker lock(&shard.mutex); auto it = shard.metadata.begin(); while (it != shard.metadata.end()) { + // CleanupStaleHandles may remove MEMORY replicas whose allocator has + // become invalid (segment unmounted). If key remains valid (has disk + // replicas), Standby must receive an updated metadata payload that + // excludes those MEMORY replicas (Scheme A). if (CleanupStaleHandles(it->second)) { - // If the object is empty, we need to erase the iterator + // No replicas remain after cleanup -> key should be deleted. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("ClearInvalidHandles(REMOVE)", + OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, it->first); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, it->first); + } +#endif it = shard.metadata.erase(it); } else { + // Still has some replicas. If HA is enabled, publish updated + // metadata WITHOUT MEMORY replicas (safe superset update). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueueLazy( + "ClearInvalidHandles(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } +#endif ++it; } } @@ -231,6 +773,9 @@ auto MasterService::ExistKey(const std::string& key) // client. metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. return true; } } @@ -387,6 +932,23 @@ auto MasterService::BatchReplicaClear( continue; } + // HA safety (Scheme A): + // This operation may free/reuse MEMORY replicas. Persist REMOVE to etcd + // BEFORE actually erasing local metadata. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("BatchReplicaClear(all)", OpType::REMOVE, + key, std::string(), + PendingMutationKind::CLEAR_ALL_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif + // Before erasing, decrement cache metrics for each COMPLETE replica for (const auto& replica : metadata.replicas) { if (replica.status() == ReplicaStatus::COMPLETE) { @@ -433,6 +995,45 @@ auto MasterService::BatchReplicaClear( continue; } + // HA safety (Scheme A): + // Removing replicas may free/reuse MEMORY replicas. Persist updated metadata + // BEFORE mutating metadata.replicas (which may free memory). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + // Build the remaining replica descriptor list after removal. + std::vector remove_mask(metadata.replicas.size(), false); + for (size_t idx : replicas_to_remove) { + if (idx < remove_mask.size()) { + remove_mask[idx] = true; + } + } + std::vector remaining; + remaining.reserve(metadata.replicas.size()); + for (size_t i = 0; i < metadata.replicas.size(); ++i) { + if (remove_mask[i]) { + continue; + } + remaining.emplace_back(metadata.replicas[i].get_descriptor()); + } + + if (remaining.empty()) { + AppendOrPersistOrEnqueue("BatchReplicaClear(partial REMOVE)", + OpType::REMOVE, key, std::string(), + PendingMutationKind::CLEAR_REPLICAS_ON_SEGMENT, + segment_name); + } else { + const std::string payload = + SerializeMetadataForOpLogFromReplicaDescriptors( + metadata.client_id, static_cast(metadata.size), + remaining); + AppendOrPersistOrEnqueue("BatchReplicaClear(partial PUT_END)", + OpType::PUT_END, key, payload, + PendingMutationKind::CLEAR_REPLICAS_ON_SEGMENT, + segment_name); + } + } +#endif + // Remove replicas on the specified segment (in reverse order to // maintain indices) for (auto it = replicas_to_remove.rbegin(); @@ -450,7 +1051,21 @@ auto MasterService::BatchReplicaClear( // If no valid replicas remain, erase the entire metadata if (metadata.replicas.empty() || !metadata.IsValid()) { +#ifndef STORE_USE_ETCD + // Non-HA: keep old behavior; HA already persisted REMOVE above. + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif accessor.Erase(); + } else { +#ifndef STORE_USE_ETCD + // Non-HA: best-effort update to keep future behavior consistent. + if (!enable_ha_) { + const std::string payload = SerializeMetadataForOpLog(metadata); + AppendOpLogAndNotify(OpType::PUT_END, key, payload); + } +#endif } cleared_keys.emplace_back(key); @@ -501,6 +1116,9 @@ auto MasterService::GetReplicaListByRegex(const std::string& regex_pattern) results.emplace(key, std::move(replica_list)); metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. } } } @@ -542,6 +1160,9 @@ auto MasterService::GetReplicaList(std::string_view key) // Grant a lease to the object so it will not be removed // when the client is reading it. metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. return GetReplicaListResponse(std::move(replica_list), default_kv_lease_ttl_); @@ -696,6 +1317,13 @@ auto MasterService::PutEnd(const UUID& client_id, const std::string& key, // at beginning. 2. If this object has soft pin enabled, set it to be soft // pinned. metadata.GrantLease(0, default_kv_soft_pin_ttl_); + + // Record OpLog entry for PUT_END so that standbys can replay this change. + // Serialize metadata (replicas, size, lease) to payload so Standby can restore + // complete metadata when promoted to Primary. + std::string metadata_payload = SerializeMetadataForOpLog(metadata); + AppendOpLogAndNotify(OpType::PUT_END, key, metadata_payload); + return {}; } @@ -719,7 +1347,7 @@ auto MasterService::AddReplica(const UUID& client_id, const std::string& key, auto& descriptor = metadata.replicas[i] .get_descriptor() .get_local_disk_descriptor(); - if (descriptor.client_id == client_id) { + if (descriptor.GetClientId() == client_id) { update = true; descriptor.transport_endpoint = replica.get_descriptor() .get_local_disk_descriptor() @@ -760,6 +1388,23 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, return tl::make_unexpected(ErrorCode::INVALID_WRITE); } + // HA behavior: + // Do NOT block subsequent ops for the same key if etcd write fails. + // We allocate sequence_id once and retry persisting this OpLogEntry + // asynchronously if needed. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("PutRevoke", OpType::PUT_REVOKE, key, std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::PUT_REVOKE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::PUT_REVOKE, key); + } +#endif + if (replica_type == ReplicaType::MEMORY) { MasterMetricManager::instance().dec_mem_cache_nums(); } else if (replica_type == ReplicaType::DISK) { @@ -776,6 +1421,7 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, if (metadata.IsValid() == false) { accessor.Erase(); } + return {}; } @@ -819,8 +1465,25 @@ auto MasterService::Remove(const std::string& key) return tl::make_unexpected(ErrorCode::REPLICA_IS_NOT_READY); } - // Remove object metadata + // HA behavior: + // If etcd write fails, enqueue retry but still proceed with local remove. + // Standby will handle gaps via timeout + late-arrival policy. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("Remove", OpType::REMOVE, key, std::string(), + PendingMutationKind::CLEAR_ALL_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif + + // Remove object metadata (may deallocate memory replicas) accessor.Erase(); + return {}; } @@ -1268,9 +1931,39 @@ void MasterService::BatchEvict(double evict_ratio_target, continue; } if (it->second.lease_timeout <= target_timeout) { - // Evict this object + // Evict this object (MEMORY replicas only). + // + // Scheme A: + // - If key remains valid after removing MEMORY replicas, + // durably persist a PUT_END carrying the updated metadata + // (without MEMORY replicas) before freeing memory. + // - If key becomes invalid (only had MEMORY replicas), + // durably persist REMOVE before freeing memory. total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { @@ -1332,9 +2025,32 @@ void MasterService::BatchEvict(double evict_ratio_target, !it->second.HasDiffRepStatus(ReplicaStatus::COMPLETE, ReplicaType::MEMORY) && it->second.HasMemReplica()) { - // Evict this object + // Evict this object (MEMORY replicas only). See Scheme A above. total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { @@ -1388,6 +2104,29 @@ void MasterService::BatchEvict(double evict_ratio_target, it->second.lease_timeout <= soft_target_timeout) { total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { @@ -1559,4 +2298,10 @@ std::string MasterService::ResolvePath(const std::string& key) const { return full_path.lexically_normal().string(); } +OpLogManager& MasterService::GetOpLogManager() { + return oplog_manager_; +} + +// SetReplicationService removed - using etcd-based OpLog sync instead + } // namespace mooncake \ No newline at end of file diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp new file mode 100644 index 0000000000..2bb089c921 --- /dev/null +++ b/mooncake-store/src/oplog_applier.cpp @@ -0,0 +1,587 @@ +#include "oplog_applier.h" + +#include +#include + +#include +#include + +#include "etcd_oplog_store.h" +#include "ha_metric_manager.h" +#include "metadata_store.h" +#include "oplog_manager.h" + +namespace mooncake { + +OpLogApplier::OpLogApplier(MetadataStore* metadata_store, + const std::string& cluster_id) + : metadata_store_(metadata_store), + cluster_id_(cluster_id), + expected_sequence_id_(1) { + if (metadata_store_ == nullptr) { + LOG(FATAL) << "OpLogApplier: metadata_store cannot be null"; + } + + // Validate cluster_id if provided (required for etcd operations). + // Normalize by stripping trailing slashes for validation. + std::string normalized = cluster_id_; + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + if (!normalized.empty() && !IsValidClusterIdComponent(normalized)) { + LOG(FATAL) << "Invalid cluster_id for OpLogApplier: '" << cluster_id_ + << "' (normalized: '" << normalized + << "'). Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } +} + +EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const { +#ifdef STORE_USE_ETCD + if (cluster_id_.empty()) { + return nullptr; + } + + std::lock_guard lock(etcd_oplog_store_mutex_); + if (!etcd_oplog_store_) { + // Reader: do not start `/latest` batch update thread. + etcd_oplog_store_ = + std::make_unique(cluster_id_, /*enable_latest_seq_batch_update=*/false); + } + return etcd_oplog_store_.get(); +#else + return nullptr; +#endif +} + +bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { + // Basic DoS protection: validate key/payload sizes before parsing/applying. + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLogApplier: entry size rejected, sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key << ", reason=" << size_reason; + return false; + } + + // Verify checksum to detect data corruption or tampering. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLogApplier: checksum mismatch, sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key + << ". Possible data corruption or tampering. Discarding entry."; + HAMetricManager::instance().inc_oplog_checksum_failures(); + return false; + } + + // Global ordering only. + // + // IMPORTANT: + // - Watch callbacks / retries may deliver duplicate or already-applied entries. + // - Those must be treated as no-op, not as "out-of-order pending", otherwise + // pending_entries_ can grow and the applier may appear stuck. + const uint64_t expected = expected_sequence_id_.load(); + if (IsSequenceOlder(entry.sequence_id, expected)) { + // Late arrival of a previously-skipped gap entry: apply only if it's a delete/revoke. + bool was_skipped = false; + { + std::lock_guard lock(pending_mutex_); + auto it = skipped_sequence_ids_.find(entry.sequence_id); + if (it != skipped_sequence_ids_.end()) { + was_skipped = true; + skipped_sequence_ids_.erase(it); + } + } + if (was_skipped) { + if (entry.op_type == OpType::REMOVE || entry.op_type == OpType::PUT_REVOKE) { + // Safe: ensure we don't keep stale metadata. + if (entry.op_type == OpType::REMOVE) { + ApplyRemove(entry); + } else { + ApplyPutRevoke(entry); + } + return true; + } + // PUT_END (or others): discard to avoid resurrecting stale state. + VLOG(1) << "OpLogApplier: discard late skipped entry, op_type=" + << static_cast(entry.op_type) + << ", sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key; + return true; + } + + VLOG(2) << "OpLogApplier: skip already-applied entry, sequence_id=" + << entry.sequence_id << ", expected=" << expected + << ", key=" << entry.object_key; + return true; // consumed (no-op) + } + if (IsSequenceNewer(entry.sequence_id, expected)) { + // Future entry - store into pending, wait for the gap to be filled. + std::lock_guard lock(pending_mutex_); + + if (pending_entries_.size() >= static_cast(kMaxPendingEntries)) { + LOG(ERROR) << "OpLogApplier: too many pending entries (" + << pending_entries_.size() << "), discarding entry sequence_id=" + << entry.sequence_id << ", key=" << entry.object_key; + return false; + } + + pending_entries_[entry.sequence_id] = entry; + VLOG(1) << "OpLogApplier: future entry buffered, sequence_id=" + << entry.sequence_id << ", expected=" << expected + << ", key=" << entry.object_key + << ", pending_entries=" << pending_entries_.size(); + return false; + } + + // Apply the operation based on type + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + default: + LOG(ERROR) << "OpLogApplier: unsupported op_type=" + << static_cast(entry.op_type) + << ", sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key; + return false; + } + + // Update expected sequence ID + expected_sequence_id_.store(entry.sequence_id + 1); + + // Update metrics + HAMetricManager::instance().inc_oplog_applied_entries(); + HAMetricManager::instance().set_oplog_applied_sequence_id( + static_cast(entry.sequence_id)); + + // Try to process pending entries + ProcessPendingEntries(); + + return true; +} + +size_t OpLogApplier::ApplyOpLogEntries(const std::vector& entries) { + size_t applied_count = 0; + for (const auto& entry : entries) { + if (ApplyOpLogEntry(entry)) { + applied_count++; + } + } + return applied_count; +} + +uint64_t OpLogApplier::GetKeySequenceId(const std::string& key) const { + // Global sequence_id is used for ordering. + (void)key; // Suppress unused parameter warning + return 0; +} + +uint64_t OpLogApplier::GetExpectedSequenceId() const { + return expected_sequence_id_.load(); +} + +void OpLogApplier::Recover(uint64_t last_applied_sequence_id) { + expected_sequence_id_.store(last_applied_sequence_id + 1); + LOG(INFO) << "OpLogApplier: recovered from sequence_id=" + << last_applied_sequence_id + << ", expected_sequence_id set to=" << expected_sequence_id_.load(); +} + +size_t OpLogApplier::ProcessPendingEntries() { + // Check for missing sequence IDs, possibly skip after timeout, and/or request them. + uint64_t missing_seq_to_request = 0; + uint64_t skipped_count = 0; + { + std::lock_guard lock(pending_mutex_); + auto now = std::chrono::steady_clock::now(); + for (;;) { + if (pending_entries_.empty()) { + break; + } + const uint64_t first_pending_seq = pending_entries_.begin()->first; + const uint64_t expected = expected_sequence_id_.load(); + if (IsSequenceOlderOrEqual(first_pending_seq, expected)) { + break; + } + + // There's a gap: expected is missing. + const uint64_t missing_seq = expected; + auto it = missing_sequence_ids_.find(missing_seq); + if (it == missing_sequence_ids_.end()) { + missing_sequence_ids_[missing_seq] = now; + ScheduleWaitForMissingEntries(missing_seq); + break; + } + + const auto waited = std::chrono::duration_cast(now - it->second); + + // Skip after timeout to avoid global stall (user requested behavior). + if (waited.count() >= kMissingEntrySkipSeconds) { + skipped_sequence_ids_[missing_seq] = now; + missing_sequence_ids_.erase(missing_seq); + expected_sequence_id_.store(missing_seq + 1); + skipped_count++; + HAMetricManager::instance().inc_oplog_skipped_entries(); + LOG(WARNING) << "OpLogApplier: skipped missing entry seq=" << missing_seq + << " after " << waited.count() << "s timeout"; + continue; // may skip multiple consecutive gaps + } + + // Best-effort request from etcd (before skip triggers). + if (waited.count() >= kMissingEntryRequestSeconds) { + missing_seq_to_request = missing_seq; + break; + } + break; + } + } + + // Request missing OpLog if needed (outside the lock to avoid deadlock) + bool retrieved_missing = false; + if (missing_seq_to_request > 0) { + retrieved_missing = RequestMissingOpLog(missing_seq_to_request); + if (retrieved_missing) { + std::lock_guard lock(pending_mutex_); + missing_sequence_ids_.erase(missing_seq_to_request); + } + } + + size_t processed_count = 0; + for (;;) { + OpLogEntry entry_copy; + bool has_entry = false; + + { + std::lock_guard lock(pending_mutex_); + if (pending_entries_.empty()) { + break; + } + + auto it = pending_entries_.begin(); + const uint64_t expected = expected_sequence_id_.load(); + if (!IsSequenceEqual(it->first, expected)) { + break; // still waiting for earlier sequence_id + } + + entry_copy = it->second; + pending_entries_.erase(it); + has_entry = true; + } + + if (!has_entry) { + break; + } + + // Apply outside lock. + switch (entry_copy.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry_copy); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry_copy); + break; + case OpType::REMOVE: + ApplyRemove(entry_copy); + break; + default: + LOG(ERROR) << "OpLogApplier: unsupported op_type in pending entry"; + break; + } + + expected_sequence_id_.store(entry_copy.sequence_id + 1); + + { + std::lock_guard lock(pending_mutex_); + missing_sequence_ids_.erase(entry_copy.sequence_id); + } + + processed_count++; + } + + // Clean up old missing sequence IDs (older than 1 minute) + { + std::lock_guard lock(pending_mutex_); + auto now = std::chrono::steady_clock::now(); + for (auto it = missing_sequence_ids_.begin(); it != missing_sequence_ids_.end();) { + auto age = std::chrono::duration_cast(now - it->second); + if (age.count() > 60) { + LOG(WARNING) << "OpLogApplier: giving up on missing sequence_id=" + << it->first << " after " << age.count() << " seconds"; + it = missing_sequence_ids_.erase(it); + } else { + ++it; + } + } + + // Clean up old skipped sequence IDs too (avoid unbounded growth). + for (auto it = skipped_sequence_ids_.begin(); it != skipped_sequence_ids_.end();) { + auto age = std::chrono::duration_cast(now - it->second); + if (age.count() > 60) { + it = skipped_sequence_ids_.erase(it); + } else { + ++it; + } + } + } + + if (skipped_count > 0) { + LOG(WARNING) << "OpLogApplier: skipped " << skipped_count + << " missing sequence_id(s) after timeout, expected_sequence_id now=" + << expected_sequence_id_.load(); + } + + if (processed_count > 0) { + LOG(INFO) << "OpLogApplier: processed " << processed_count + << " pending entries, expected_sequence_id now=" + << expected_sequence_id_.load(); + } + + // Update pending entries metric + { + std::lock_guard lock(pending_mutex_); + HAMetricManager::instance().set_oplog_pending_entries( + static_cast(pending_entries_.size())); + } + + return processed_count; +} + +OpLogApplier::GapResolveResult OpLogApplier::TryResolveGapsOnceForPromotion( + size_t max_ids) { + GapResolveResult r; +#ifdef STORE_USE_ETCD + EtcdOpLogStore* store = GetEtcdOpLogStore(); + if (store == nullptr) { + return r; + } + + std::vector gap_ids; + gap_ids.reserve(max_ids); + { + std::lock_guard lock(pending_mutex_); + for (const auto& kv : missing_sequence_ids_) { + if (gap_ids.size() >= max_ids) break; + gap_ids.push_back(kv.first); + } + for (const auto& kv : skipped_sequence_ids_) { + if (gap_ids.size() >= max_ids) break; + gap_ids.push_back(kv.first); + } + } + + if (gap_ids.empty()) { + return r; + } + + std::sort(gap_ids.begin(), gap_ids.end()); + gap_ids.erase(std::unique(gap_ids.begin(), gap_ids.end()), gap_ids.end()); + + r.attempted = gap_ids.size(); + std::vector successfully_processed; + for (uint64_t seq : gap_ids) { + OpLogEntry e; + ErrorCode err = store->ReadOpLog(seq, e); + if (err != ErrorCode::OK) { + // Log failed gap for monitoring, but don't clear it so it can be retried later. + LOG(WARNING) << "Promotion gap resolve: failed to fetch seq=" << seq + << ", err=" << static_cast(err); + continue; + } + r.fetched++; + + // Apply policy: only delete/revoke; drop PUT_END. + if (e.op_type == OpType::REMOVE) { + ApplyRemove(e); + r.applied_deletes++; + successfully_processed.push_back(seq); + } else if (e.op_type == OpType::PUT_REVOKE) { + ApplyPutRevoke(e); + r.applied_deletes++; + successfully_processed.push_back(seq); + } else { + // PUT_END or others: mark as processed (dropped) so we don't retry. + successfully_processed.push_back(seq); + } + } + + // Only clear gaps we successfully fetched and processed. + // Failed gaps remain in missing_sequence_ids_/skipped_sequence_ids_ for potential + // retry or monitoring. + if (!successfully_processed.empty()) { + std::lock_guard lock(pending_mutex_); + for (uint64_t seq : successfully_processed) { + missing_sequence_ids_.erase(seq); + skipped_sequence_ids_.erase(seq); + } + } + return r; +#else + (void)max_ids; + return r; +#endif +} + +bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { + // Only check global sequence order. + // Use IsSequenceEqual for wrap-around safety (though equality check doesn't + // need special handling, we use it for consistency). + return IsSequenceEqual(entry.sequence_id, expected_sequence_id_.load()); +} + +void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) { + // Payload contains serialized metadata (replicas, size, etc.) in JSON format. + // Deserialize the payload immediately and store structured metadata. + // This allows Standby to serve requests immediately after promotion. + + if (entry.payload.empty()) { + // No payload - create empty metadata (legacy compatibility) + LOG(WARNING) << "OpLogApplier: PUT_END without payload, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + StandbyObjectMetadata empty_metadata; + empty_metadata.last_sequence_id = entry.sequence_id; + if (!metadata_store_->PutMetadata(entry.object_key, empty_metadata)) { + LOG(ERROR) << "OpLogApplier: failed to PutMetadata key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } + return; + } + + // Deserialize payload to MetadataPayload + MetadataPayload payload; + bool parse_success = false; + try { + struct_json::from_json(payload, entry.payload); + parse_success = true; + } catch (const std::exception& e) { + const std::string prefix = + entry.payload.size() > 256 ? entry.payload.substr(0, 256) : entry.payload; + LOG(ERROR) << "OpLogApplier: failed to parse payload for key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << ", payload_size=" << entry.payload.size() + << ", payload_prefix(256)=" << prefix + << ", error=" << e.what(); + } + + if (!parse_success) { + // Fallback to empty metadata if parsing fails + StandbyObjectMetadata empty_metadata; + empty_metadata.last_sequence_id = entry.sequence_id; + metadata_store_->PutMetadata(entry.object_key, empty_metadata); + return; + } + + // Convert to StandbyObjectMetadata and store + StandbyObjectMetadata metadata = payload.ToStandbyMetadata(entry.sequence_id); + + if (!metadata_store_->PutMetadata(entry.object_key, metadata)) { + LOG(ERROR) << "OpLogApplier: failed to PutMetadata key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } else { + VLOG(1) << "OpLogApplier: applied PUT_END, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << ", replicas=" << metadata.replicas.size() + << ", size=" << metadata.size; + } +} + +void OpLogApplier::ApplyPutRevoke(const OpLogEntry& entry) { + // PUT_REVOKE means the object should be removed from metadata store + // (but the key itself may still exist if there are other replicas). + // Current implementation removes the entire key; if we later support + // partial replica revocation this logic will need to be refined. + if (!metadata_store_->Remove(entry.object_key)) { + LOG(WARNING) << "OpLogApplier: failed to Remove key=" << entry.object_key + << " in PUT_REVOKE, sequence_id=" << entry.sequence_id + << " (key may not exist)"; + } else { + VLOG(1) << "OpLogApplier: applied PUT_REVOKE, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } +} + +void OpLogApplier::ApplyRemove(const OpLogEntry& entry) { + if (!metadata_store_->Remove(entry.object_key)) { + LOG(WARNING) << "OpLogApplier: failed to Remove key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << " (key may not exist)"; + } else { + VLOG(1) << "OpLogApplier: applied REMOVE, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } +} + +bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) { +#ifdef STORE_USE_ETCD + HAMetricManager::instance().inc_oplog_gap_resolve_attempts(); + + EtcdOpLogStore* oplog_store = GetEtcdOpLogStore(); + if (oplog_store == nullptr) { + LOG(WARNING) << "OpLogApplier: cannot request missing OpLog, cluster_id not set"; + return false; + } + + OpLogEntry entry; + ErrorCode err = oplog_store->ReadOpLog(missing_seq_id, entry); + if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { + LOG(INFO) << "OpLogApplier: missing OpLog entry not found in etcd, sequence_id=" + << missing_seq_id; + return false; + } + if (err != ErrorCode::OK) { + LOG(ERROR) << "OpLogApplier: failed to read missing OpLog from etcd, sequence_id=" + << missing_seq_id << ", error=" << static_cast(err); + return false; + } + + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLogApplier: missing entry size rejected, sequence_id=" + << missing_seq_id << ", key=" << entry.object_key + << ", reason=" << size_reason; + return false; + } + + // Verify checksum before adding to pending entries. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLogApplier: checksum mismatch for retrieved missing entry, sequence_id=" + << missing_seq_id << ", key=" << entry.object_key + << ". Possible data corruption. Discarding entry."; + HAMetricManager::instance().inc_oplog_checksum_failures(); + return false; + } + + // Successfully retrieved the missing OpLog entry + LOG(INFO) << "OpLogApplier: retrieved missing OpLog entry, sequence_id=" + << missing_seq_id << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + HAMetricManager::instance().inc_oplog_gap_resolve_success(); + + // Add to pending entries + // Note: We don't call ProcessPendingEntries() here to avoid potential recursion. + // The caller (ProcessPendingEntries itself) will process the entry in the next loop. + { + std::lock_guard lock(pending_mutex_); + pending_entries_[entry.sequence_id] = entry; + } + + return true; +#else + LOG(WARNING) << "OpLogApplier: STORE_USE_ETCD not enabled, cannot request missing OpLog"; + return false; +#endif +} + +void OpLogApplier::ScheduleWaitForMissingEntries(uint64_t missing_seq_id) { + // This method is called when we first detect a missing sequence_id. + // The actual waiting and requesting is handled in ProcessPendingEntries(). + // We just log it here for tracking. + VLOG(1) << "OpLogApplier: scheduling wait for missing sequence_id=" << missing_seq_id + << ", will request after " << kMissingEntryRequestSeconds << " seconds"; +} + +} // namespace mooncake + diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp new file mode 100644 index 0000000000..17d4da84ba --- /dev/null +++ b/mooncake-store/src/oplog_manager.cpp @@ -0,0 +1,171 @@ +#include "oplog_manager.h" + +#include +#include +#include +#include + +#include "etcd_oplog_store.h" + +namespace mooncake { + +OpLogManager::OpLogManager() = default; + +void OpLogManager::SetEtcdOpLogStore( + std::shared_ptr etcd_oplog_store) { + std::unique_lock lock(mutex_); + etcd_oplog_store_ = etcd_oplog_store; +} + +uint64_t OpLogManager::Append(OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry entry; + entry.op_type = type; + entry.object_key = key; + entry.payload = payload; + entry.timestamp_ms = NowMs(); + entry.checksum = ComputeChecksum(entry.payload); + entry.prefix_hash = ComputePrefixHash(entry.object_key); + + std::unique_lock lock(mutex_); + entry.sequence_id = ++last_seq_id_; + + if (buffer_.size() >= kMaxBufferEntries_) { + buffer_.pop_front(); + ++first_seq_id_; + } + + buffer_.emplace_back(entry); // Copy entry to buffer + + // Write to etcd if EtcdOpLogStore is set + if (etcd_oplog_store_) { + // Release lock before writing to etcd to avoid blocking + // We use the original entry (before it was copied to buffer) + lock.unlock(); + ErrorCode err = etcd_oplog_store_->WriteOpLog(entry); + if (err != ErrorCode::OK) { + // Log error but don't fail the operation + // The entry is already in the memory buffer + LOG(WARNING) << "Failed to write OpLog to etcd, sequence_id=" + << entry.sequence_id + << ", but entry is in memory buffer"; + } + } + + return last_seq_id_; +} + +OpLogEntry OpLogManager::AllocateEntry(OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry entry; + entry.op_type = type; + entry.object_key = key; + entry.payload = payload; + entry.timestamp_ms = NowMs(); + entry.checksum = ComputeChecksum(entry.payload); + entry.prefix_hash = ComputePrefixHash(entry.object_key); + + std::unique_lock lock(mutex_); + entry.sequence_id = ++last_seq_id_; + + if (buffer_.size() >= kMaxBufferEntries_) { + buffer_.pop_front(); + ++first_seq_id_; + } + buffer_.emplace_back(entry); + return entry; +} + +ErrorCode OpLogManager::PersistEntryToEtcd(const OpLogEntry& entry) const { + std::shared_lock lock(mutex_); + auto store = etcd_oplog_store_; + lock.unlock(); + if (!store) { + return ErrorCode::ETCD_OPERATION_ERROR; + } + return store->WriteOpLog(entry); +} + +tl::expected OpLogManager::AppendAndPersist( + OpType type, const std::string& key, const std::string& payload) { + // Seq pre-allocation semantics: allocate first, then persist. + OpLogEntry entry = AllocateEntry(type, key, payload); + ErrorCode err = PersistEntryToEtcd(entry); + if (err != ErrorCode::OK) { + return tl::make_unexpected(err); + } + return entry.sequence_id; +} + +uint64_t OpLogManager::GetLastSequenceId() const { + std::shared_lock lock(mutex_); + return last_seq_id_; +} + +void OpLogManager::SetInitialSequenceId(uint64_t sequence_id) { + std::unique_lock lock(mutex_); + if (last_seq_id_ == 0 && buffer_.empty()) { + // Only allow setting initial sequence_id if OpLogManager is empty + last_seq_id_ = sequence_id; + first_seq_id_ = sequence_id + 1; // first_seq_id_ should be > last_seq_id_ when empty + LOG(INFO) << "OpLogManager initial sequence_id set to " << sequence_id; + } else { + LOG(WARNING) << "Cannot set initial sequence_id: OpLogManager is not empty " + << "(last_seq_id_=" << last_seq_id_ << ", buffer_size=" << buffer_.size() << ")"; + } +} + +size_t OpLogManager::GetEntryCount() const { + std::shared_lock lock(mutex_); + return buffer_.size(); +} + +uint64_t OpLogManager::NowMs() { + using namespace std::chrono; + return duration_cast(steady_clock::now().time_since_epoch()) + .count(); +} + +uint32_t OpLogManager::ComputeChecksum(const std::string& data) { + // Use xxHash XXH32 for a fast, deterministic 32-bit checksum. + // Requires linking against xxHash (e.g., libxxhash) and including . + return static_cast(XXH32(data.data(), data.size(), 0)); +} + +uint32_t OpLogManager::ComputePrefixHash(const std::string& key) { + if (key.empty()) { + return 0; + } + // Use XXH32 for consistency with ComputeChecksum and better performance. + // XXH32 provides faster hashing and lower collision rate than std::hash. + // Computing hash for the entire key ensures better distribution and fewer collisions. + return static_cast(XXH32(key.data(), key.size(), 0)); +} + +bool OpLogManager::VerifyChecksum(const OpLogEntry& entry) { + uint32_t computed = ComputeChecksum(entry.payload); + return computed == entry.checksum; +} + +bool OpLogManager::ValidateEntrySize(const OpLogEntry& entry, + std::string* reason) { + if (entry.object_key.size() > kMaxObjectKeySize) { + if (reason) { + *reason = "object_key too large: size=" + + std::to_string(entry.object_key.size()); + } + return false; + } + if (entry.payload.size() > kMaxPayloadSize) { + if (reason) { + *reason = + "payload too large: size=" + std::to_string(entry.payload.size()); + } + return false; + } + return true; +} + +} // namespace mooncake + + diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp new file mode 100644 index 0000000000..f3b7a2bb32 --- /dev/null +++ b/mooncake-store/src/oplog_watcher.cpp @@ -0,0 +1,490 @@ +#include "oplog_watcher.h" + +#include +#include +#include +#include +#include + +#ifdef STORE_USE_ETCD +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "ha_metric_manager.h" +#include "oplog_applier.h" +#include "oplog_manager.h" + +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif + +namespace mooncake { + +OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), cluster_id_(cluster_id), applier_(applier) { + if (applier_ == nullptr) { + LOG(FATAL) << "OpLogApplier cannot be null"; + } + // Normalize cluster_id to avoid double slashes in watch prefix. + while (!cluster_id_.empty() && cluster_id_.back() == '/') { + cluster_id_.pop_back(); + } + if (!cluster_id_.empty() && !IsValidClusterIdComponent(cluster_id_)) { + LOG(FATAL) << "Invalid cluster_id for OpLogWatcher: '" << cluster_id_ + << "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } +} + +OpLogWatcher::~OpLogWatcher() { + Stop(); +} + +void OpLogWatcher::Start() { + // Backward-compatible: start from the last processed sequence id. + (void)StartFromSequenceId(last_processed_sequence_id_.load()); +} + +bool OpLogWatcher::StartFromSequenceId(uint64_t start_seq_id) { + if (running_.load()) { + LOG(WARNING) << "OpLogWatcher is already running"; + return true; + } + +#ifdef STORE_USE_ETCD + uint64_t read_seq_id = start_seq_id; + EtcdRevisionId last_read_rev = 0; + size_t total_applied = 0; + + for (;;) { + std::vector batch; + EtcdRevisionId rev = 0; + if (!ReadOpLogSince(read_seq_id, batch, rev)) { + last_read_rev = 0; + break; + } + last_read_rev = rev; + if (!batch.empty()) { + for (const auto& e : batch) { + if (applier_->ApplyOpLogEntry(e)) { + last_processed_sequence_id_.store(e.sequence_id); + read_seq_id = e.sequence_id; + total_applied++; + } + } + } + if (batch.size() < kSyncBatchSize) { + break; + } + } + + if (last_read_rev > 0) { + next_watch_revision_.store(static_cast(last_read_rev + 1)); + } else { + next_watch_revision_.store(0); + } + + LOG(INFO) << "OpLogWatcher initial sync done: applied=" << total_applied + << ", last_seq=" << last_processed_sequence_id_.load() + << ", next_watch_revision=" << next_watch_revision_.load(); +#endif + + running_.store(true); + watch_thread_ = std::thread(&OpLogWatcher::WatchOpLog, this); + LOG(INFO) << "OpLogWatcher started for cluster_id=" << cluster_id_; + return true; +} + +void OpLogWatcher::Stop() { + if (!running_.load()) { + return; + } + + running_.store(false); + +#ifdef STORE_USE_ETCD + // Cancel the watch + std::string watch_prefix = "/oplog/" + cluster_id_ + "/"; + ErrorCode err = EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to cancel watch for prefix " << watch_prefix + << ", error=" << static_cast(err); + } +#endif + + // Wait for watch thread to finish + if (watch_thread_.joinable()) { + watch_thread_.join(); + } + + LOG(INFO) << "OpLogWatcher stopped"; +} + +bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries, + EtcdRevisionId& revision_id) { +#ifdef STORE_USE_ETCD + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + ErrorCode err = oplog_store.ReadOpLogSinceWithRevision( + start_seq_id, kSyncBatchSize, entries, revision_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to read OpLog since sequence_id=" << start_seq_id + << ", error=" << static_cast(err); + return false; + } + return true; +#else + (void)start_seq_id; + (void)entries; + (void)revision_id; + return false; +#endif +} + +uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { + return last_processed_sequence_id_.load(); +} + +void OpLogWatcher::WatchCallback(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, + int event_type, int64_t mod_revision) { + OpLogWatcher* watcher = static_cast(context); + if (watcher == nullptr) { + LOG(ERROR) << "OpLogWatcher context is null"; + return; + } + + std::string key_str; + if (key != nullptr && key_size > 0) { + key_str.assign(key, key_size); + } + std::string value_str; + if (value != nullptr && value_size > 0) { + value_str = std::string(value, value_size); + } + watcher->HandleWatchEvent(key_str, value_str, event_type, mod_revision); +} + +void OpLogWatcher::WatchOpLog() { +#ifdef STORE_USE_ETCD + LOG(INFO) << "OpLog watch thread started for cluster_id=" << cluster_id_; + + std::string watch_prefix = "/oplog/" + cluster_id_ + "/"; + + while (running_.load()) { + // Start watching - pass static callback function and this pointer as context + EtcdRevisionId start_rev = + static_cast(next_watch_revision_.load()); + // Use watcher with mod_revision so we can update next_watch_revision_ precisely. + ErrorCode err = EtcdHelper::WatchWithPrefixFromRevision( + watch_prefix.c_str(), watch_prefix.size(), start_rev, this, WatchCallback); + + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start watch for prefix " << watch_prefix + << ", error=" << static_cast(err); + watch_healthy_.store(false); + NotifyStateEvent(StandbyEvent::WATCH_BROKEN); + + // Try to reconnect + TryReconnect(); + continue; + } + + LOG(INFO) << "Watch started for prefix " << watch_prefix; + watch_healthy_.store(true); + consecutive_errors_.store(0); + NotifyStateEvent(StandbyEvent::WATCH_HEALTHY); + + // The watch is now running in the background (via Go goroutine) + // We just need to keep the thread alive until Stop() is called or watch fails + while (running_.load() && watch_healthy_.load()) { + // Drive pending/missing handling even when no new watch events arrive. + // Without this, a single out-of-order arrival could park entries in + // pending_entries_ forever if the missing entry isn't delivered via watch + // (but exists in etcd and could be fetched). + (void)applier_->ProcessPendingEntries(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Periodically check watch health + if (consecutive_errors_.load() >= kMaxConsecutiveErrors) { + LOG(WARNING) << "Too many consecutive errors (" << consecutive_errors_.load() + << "), reconnecting watch..."; + watch_healthy_.store(false); + NotifyStateEvent(StandbyEvent::MAX_ERRORS_REACHED); + break; + } + } + + if (running_.load() && !watch_healthy_.load()) { + // Cancel current watch before reconnecting + EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + NotifyStateEvent(StandbyEvent::WATCH_BROKEN); + TryReconnect(); + } + } + + LOG(INFO) << "OpLog watch thread stopped"; +#else + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot watch OpLog from etcd"; + running_.store(false); +#endif +} + +void OpLogWatcher::TryReconnect() { + if (!running_.load()) { + return; + } + + int reconnect_attempt = reconnect_count_.fetch_add(1) + 1; + + // Calculate delay with exponential backoff + int delay_ms = std::min(kReconnectDelayMs * reconnect_attempt, kMaxReconnectDelayMs); + + LOG(INFO) << "Attempting to reconnect watch (attempt #" << reconnect_attempt + << "), waiting " << delay_ms << "ms..."; + + std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); + + // Sync any missed entries before resuming watch + if (SyncMissedEntries()) { + LOG(INFO) << "Successfully synced missed OpLog entries"; + NotifyStateEvent(StandbyEvent::RECOVERY_SUCCESS); + } else { + LOG(WARNING) << "Failed to sync missed OpLog entries, continuing anyway"; + NotifyStateEvent(StandbyEvent::RECOVERY_FAILED); + } +} + +bool OpLogWatcher::SyncMissedEntries() { +#ifdef STORE_USE_ETCD + uint64_t last_seq = last_processed_sequence_id_.load(); + if (last_seq == 0) { + // No entries processed yet, nothing to sync + return true; + } + + LOG(INFO) << "Syncing missed OpLog entries since sequence_id=" << last_seq; + + std::vector entries; + EtcdRevisionId rev = 0; + if (!ReadOpLogSince(last_seq, entries, rev)) { + LOG(ERROR) << "Failed to read missed OpLog entries"; + return false; + } + if (rev > 0) { + next_watch_revision_.store(static_cast(rev + 1)); + } + + if (entries.empty()) { + LOG(INFO) << "No missed OpLog entries to sync"; + return true; + } + + LOG(INFO) << "Syncing " << entries.size() << " missed OpLog entries"; + + for (const auto& entry : entries) { + if (applier_->ApplyOpLogEntry(entry)) { + last_processed_sequence_id_.store(entry.sequence_id); + } else { + LOG(WARNING) << "Failed to apply missed OpLog entry, sequence_id=" + << entry.sequence_id; + } + } + + return true; +#else + return false; +#endif +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type) { + HandleWatchEvent(key, value, event_type, /*mod_revision=*/0); +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision) { + // event_type: + // 0 = PUT, 1 = DELETE, 2 = WATCH_BROKEN (Go watcher terminated; should reconnect) + if (event_type == 2) { + LOG(WARNING) << "OpLog watch broken, will reconnect. cluster_id=" << cluster_id_ + << ", next_watch_revision=" << next_watch_revision_.load() + << ", last_seq=" << last_processed_sequence_id_.load(); + watch_healthy_.store(false); + consecutive_errors_.fetch_add(1); + return; + } + + if (mod_revision > 0) { + // Keep next_watch_revision_ monotonic: next = max(next, modRev+1) + int64_t candidate = mod_revision + 1; + int64_t cur = next_watch_revision_.load(); + while (candidate > cur && + !next_watch_revision_.compare_exchange_weak(cur, candidate)) { + // retry + } + } + // event_type: 0 = PUT, 1 = DELETE + if (event_type == 1) { + // DELETE event - OpLog entry was cleaned up + VLOG(1) << "OpLog entry deleted: " << key; + consecutive_errors_.store(0); // Watch is working + return; + } + + if (event_type != 0) { + LOG(WARNING) << "Unknown event type: " << event_type << " for key: " << key; + consecutive_errors_.fetch_add(1); + return; + } + + // Skip the "latest" key and snapshot keys + if (key.find("/latest") != std::string::npos || + key.find("/snapshot/") != std::string::npos) { + return; + } + + // Parse the OpLog entry from JSON + OpLogEntry entry; + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry from key: " << key; + consecutive_errors_.fetch_add(1); + return; + } + + // Basic DoS protection: validate key/payload sizes before further processing. + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLog entry size rejected: sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key << ", reason=" << size_reason; + consecutive_errors_.fetch_add(1); + return; + } + + // Verify checksum to detect data corruption or tampering. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLog entry checksum mismatch: sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key + << ". Possible data corruption or tampering. Discarding entry."; + consecutive_errors_.fetch_add(1); + HAMetricManager::instance().inc_oplog_checksum_failures(); + return; + } + + // Apply the OpLog entry + if (applier_->ApplyOpLogEntry(entry)) { + // last_processed_sequence_id_ must be monotonic. We may "consume" duplicate + // / already-applied entries (entry.sequence_id < expected) as no-ops, so + // never regress this counter. + uint64_t cur = last_processed_sequence_id_.load(); + while (IsSequenceNewer(entry.sequence_id, cur) && + !last_processed_sequence_id_.compare_exchange_weak(cur, entry.sequence_id)) { + // retry + } + consecutive_errors_.store(0); // Reset error counter on success + reconnect_count_.store(0); // Reset reconnect counter on success + VLOG(2) << "Applied OpLog entry: sequence_id=" << entry.sequence_id + << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + } else { + // ApplyOpLogEntry returns false for out-of-order entries, + // which is expected behavior, not an error + VLOG(1) << "OpLog entry not applied (may be out of order): sequence_id=" + << entry.sequence_id; + } +} + +bool OpLogWatcher::DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) { + Json::Value root; + Json::CharReaderBuilder reader; + std::string errs; + std::istringstream s(json_str); + + if (!Json::parseFromStream(reader, s, &root, &errs)) { + LOG(ERROR) << "Failed to parse OpLogEntry JSON: " << errs; + return false; + } + + entry.sequence_id = root.get("sequence_id", 0).asUInt64(); + entry.timestamp_ms = root.get("timestamp_ms", 0).asUInt64(); + entry.op_type = static_cast(root.get("op_type", 0).asInt()); + entry.object_key = root.get("object_key", "").asString(); + entry.payload = root.get("payload", "").asString(); + entry.checksum = root.get("checksum", 0).asUInt(); + entry.prefix_hash = root.get("prefix_hash", 0).asUInt(); + return true; +} + +} // namespace mooncake + +#else // STORE_USE_ETCD not defined + +namespace mooncake { + +OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), cluster_id_(cluster_id), applier_(applier) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +OpLogWatcher::~OpLogWatcher() { + Stop(); +} + +void OpLogWatcher::Start() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +bool OpLogWatcher::StartFromSequenceId(uint64_t /*start_seq_id*/) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +void OpLogWatcher::Stop() { + // No-op when STORE_USE_ETCD is not enabled +} + +bool OpLogWatcher::ReadOpLogSince(uint64_t /*start_seq_id*/, + std::vector& /*entries*/, + EtcdRevisionId& /*revision_id*/) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { + return last_processed_sequence_id_.load(); +} + +void OpLogWatcher::WatchOpLog() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision) { + (void)key; + (void)value; + (void)event_type; + (void)mod_revision; + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::TryReconnect() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +bool OpLogWatcher::SyncMissedEntries() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +} // namespace mooncake + +#endif // STORE_USE_ETCD + diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index db1cec0cad..7b33698814 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -13,12 +13,14 @@ #include #include +#include "ha_metric_manager.h" #include "master_metric_manager.h" #include "master_service.h" #include "rpc_helper.h" #include "types.h" #include "utils/scoped_vlog_timer.h" #include "version.h" +// replication_service.h removed - using etcd-based OpLog sync instead namespace mooncake { @@ -31,12 +33,24 @@ WrappedMasterService::WrappedMasterService( metric_report_running_(config.enable_metric_reporting) { init_http_server(); + // ReplicationService removed - using etcd-based OpLog sync instead + // TODO: In Phase 1, initialize EtcdOpLogStore and integrate with + // OpLogManager + if (config.enable_ha) { + LOG(INFO) << "HA mode enabled - etcd-based OpLog sync will be " + "implemented in Phase 1"; + } + if (config.enable_metric_reporting) { metric_report_thread_ = std::thread([this]() { while (metric_report_running_) { std::string metrics_summary = MasterMetricManager::instance().get_summary_string(); LOG(INFO) << "Master Metrics: " << metrics_summary; + // Log HA metrics summary + std::string ha_summary = + HAMetricManager::instance().get_summary_string(); + LOG(INFO) << ha_summary; std::this_thread::sleep_for( std::chrono::seconds(kMetricReportIntervalSeconds)); } @@ -49,9 +63,19 @@ WrappedMasterService::~WrappedMasterService() { if (metric_report_thread_.joinable()) { metric_report_thread_.join(); } + + // ReplicationService removed - using etcd-based OpLog sync instead + http_server_.stop(); } +void WrappedMasterService::RestoreFromStandby( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id) { + master_service_.RestoreFromStandbySnapshot(snapshot, + initial_oplog_sequence_id); +} + void WrappedMasterService::init_http_server() { using namespace coro_http; @@ -59,6 +83,8 @@ void WrappedMasterService::init_http_server() { "/metrics", [](coro_http_request& req, coro_http_response& resp) { std::string metrics = MasterMetricManager::instance().serialize_metrics(); + // Append HA metrics + metrics += HAMetricManager::instance().serialize_metrics(); resp.add_header("Content-Type", "text/plain; version=0.0.4"); resp.set_status_and_content(status_type::ok, std::move(metrics)); }); @@ -68,10 +94,21 @@ void WrappedMasterService::init_http_server() { [](coro_http_request& req, coro_http_response& resp) { std::string summary = MasterMetricManager::instance().get_summary_string(); + summary += "\n"; + summary += HAMetricManager::instance().get_summary_string(); resp.add_header("Content-Type", "text/plain; version=0.0.4"); resp.set_status_and_content(status_type::ok, std::move(summary)); }); + // Dedicated HA metrics endpoint + http_server_.set_http_handler( + "/metrics/ha", [](coro_http_request& req, coro_http_response& resp) { + std::string metrics = + HAMetricManager::instance().serialize_metrics(); + resp.add_header("Content-Type", "text/plain; version=0.0.4"); + resp.set_status_and_content(status_type::ok, std::move(metrics)); + }); + http_server_.set_http_handler( "/query_key", [&](coro_http_request& req, coro_http_response& resp) { auto key = req.get_query_value("key"); diff --git a/mooncake-store/src/standby_state_machine.cpp b/mooncake-store/src/standby_state_machine.cpp new file mode 100644 index 0000000000..616649a454 --- /dev/null +++ b/mooncake-store/src/standby_state_machine.cpp @@ -0,0 +1,283 @@ +#include "standby_state_machine.h" + +#include + +namespace mooncake { + +StandbyStateMachine::StandbyStateMachine() + : state_enter_time_(std::chrono::steady_clock::now()) {} + +StateTransitionResult StandbyStateMachine::ValidateTransition(StandbyState from, + StandbyEvent event) const { + StateTransitionResult result; + result.allowed = false; + result.old_state = from; + result.new_state = from; + + // State transition table + switch (from) { + case StandbyState::STOPPED: + if (event == StandbyEvent::START) { + result.allowed = true; + result.new_state = StandbyState::CONNECTING; + } + break; + + case StandbyState::CONNECTING: + switch (event) { + case StandbyEvent::CONNECTED: + result.allowed = true; + result.new_state = StandbyState::SYNCING; + break; + case StandbyEvent::CONNECTION_FAILED: + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::SYNCING: + switch (event) { + case StandbyEvent::SYNC_COMPLETE: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + case StandbyEvent::SYNC_FAILED: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + default: + break; + } + break; + + case StandbyState::WATCHING: + switch (event) { + case StandbyEvent::WATCH_BROKEN: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::MAX_ERRORS_REACHED: + result.allowed = true; + result.new_state = StandbyState::RECOVERING; + break; + case StandbyEvent::PROMOTE: + result.allowed = true; + result.new_state = StandbyState::PROMOTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + // WATCH_HEALTHY in WATCHING state is a no-op (stay in WATCHING) + case StandbyEvent::WATCH_HEALTHY: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + default: + break; + } + break; + + case StandbyState::RECOVERING: + switch (event) { + case StandbyEvent::RECOVERY_SUCCESS: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + case StandbyEvent::RECOVERY_FAILED: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + default: + break; + } + break; + + case StandbyState::RECONNECTING: + switch (event) { + case StandbyEvent::CONNECTED: + result.allowed = true; + result.new_state = StandbyState::SYNCING; + break; + case StandbyEvent::MAX_ERRORS_REACHED: + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::PROMOTING: + switch (event) { + case StandbyEvent::PROMOTION_SUCCESS: + result.allowed = true; + result.new_state = StandbyState::PROMOTED; + break; + case StandbyEvent::PROMOTION_FAILED: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::PROMOTED: + if (event == StandbyEvent::STOP) { + result.allowed = true; + result.new_state = StandbyState::STOPPED; + } + break; + + case StandbyState::FAILED: + if (event == StandbyEvent::STOP) { + result.allowed = true; + result.new_state = StandbyState::STOPPED; + } else if (event == StandbyEvent::START) { + // Allow restart from FAILED state + result.allowed = true; + result.new_state = StandbyState::CONNECTING; + } + break; + } + + if (!result.allowed) { + result.reason = std::string("Invalid transition from ") + StandbyStateToString(from) + + " on event " + StandbyEventToString(event); + } + + return result; +} + +StateTransitionResult StandbyStateMachine::ProcessEvent(StandbyEvent event) { + StandbyState old_state = current_state_.load(std::memory_order_acquire); + StateTransitionResult result = ValidateTransition(old_state, event); + + if (result.allowed && result.new_state != old_state) { + std::lock_guard lock(mutex_); + + // Double-check state hasn't changed (compare-and-swap pattern) + StandbyState current = current_state_.load(std::memory_order_acquire); + if (current != old_state) { + // State changed by another thread, re-validate + result = ValidateTransition(current, event); + old_state = current; + result.old_state = current; + if (!result.allowed || result.new_state == old_state) { + return result; + } + } + + // Record transition + TransitionRecord record; + record.timestamp = std::chrono::steady_clock::now(); + record.from_state = old_state; + record.to_state = result.new_state; + record.event = event; + + transition_history_.push_back(record); + if (transition_history_.size() > kMaxHistorySize) { + transition_history_.erase(transition_history_.begin()); + } + + // Update state + current_state_.store(result.new_state, std::memory_order_release); + state_enter_time_ = record.timestamp; + + LOG(INFO) << "Standby state transition: " << StandbyStateToString(old_state) << " -> " + << StandbyStateToString(result.new_state) + << " (event: " << StandbyEventToString(event) << ")"; + + // Make a copy of callbacks to release the lock before calling them + std::vector callbacks_copy = callbacks_; + + // Release lock by ending scope, then notify callbacks + // Note: We need to unlock before calling callbacks to avoid deadlock + // So we copy callbacks and call after the lock_guard scope ends + for (const auto& callback : callbacks_copy) { + if (callback) { + callback(old_state, result.new_state, event); + } + } + } else if (!result.allowed) { + VLOG(1) << "Standby state transition rejected: " << result.reason; + } + + return result; +} + +void StandbyStateMachine::RegisterCallback(StateChangeCallback callback) { + std::lock_guard lock(mutex_); + callbacks_.push_back(std::move(callback)); +} + +std::vector StandbyStateMachine::GetTransitionHistory( + size_t max_records) const { + std::lock_guard lock(mutex_); + + if (transition_history_.size() <= max_records) { + return transition_history_; + } + + return std::vector(transition_history_.end() - max_records, + transition_history_.end()); +} + +std::chrono::milliseconds StandbyStateMachine::GetTimeInCurrentState() const { + auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(mutex_); + return std::chrono::duration_cast(now - state_enter_time_); +} + +int StandbyStateMachine::IncrementErrors() { + int new_count = consecutive_errors_.fetch_add(1) + 1; + if (new_count >= kMaxConsecutiveErrors) { + // Trigger MAX_ERRORS_REACHED event + ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + } + return new_count; +} + +} // namespace mooncake + diff --git a/mooncake-store/tests/CMakeLists.txt b/mooncake-store/tests/CMakeLists.txt index 15c838e190..21d45f1d73 100644 --- a/mooncake-store/tests/CMakeLists.txt +++ b/mooncake-store/tests/CMakeLists.txt @@ -1,5 +1,10 @@ function(add_store_test name) add_executable(${name} ${ARGN}) + # Set include directories for tests + target_include_directories(${name} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../include + ${CMAKE_CURRENT_BINARY_DIR}/../include + ) target_link_libraries(${name} PUBLIC mooncake_store cachelib_memory_allocator @@ -35,6 +40,10 @@ add_store_test(non_ha_reconnect_test non_ha_reconnect_test.cpp) add_store_test(storage_backend_test storage_backend_test.cpp) add_store_test(mutex_test mutex_test.cpp) add_store_test(file_storage_test file_storage_test.cpp) + +# Hot Standby Unit Tests +add_store_test(standby_state_machine_test hot_standby_ut/standby_state_machine_test.cpp) + add_subdirectory(e2e) add_executable(high_availability_test high_availability_test.cpp) diff --git a/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp b/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp new file mode 100644 index 0000000000..2d39f64122 --- /dev/null +++ b/mooncake-store/tests/hot_standby_ut/standby_state_machine_test.cpp @@ -0,0 +1,729 @@ +#include "standby_state_machine.h" + +#include +#include + +#include +#include +#include +#include + +namespace mooncake::test { + +class StandbyStateMachineTest : public ::testing::Test { + protected: + void SetUp() override { + google::InitGoogleLogging("StandbyStateMachineTest"); + FLAGS_logtostderr = true; + machine_ = std::make_unique(); + } + + void TearDown() override { google::ShutdownGoogleLogging(); } + + std::unique_ptr machine_; + + // Helper function to reach WATCHING state + void ReachWatchingState() { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); + } + + // Helper function to reach SYNCING state + void ReachSyncingState() { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + } +}; + +// ========== Initial State Tests ========== + +TEST_F(StandbyStateMachineTest, TestInitialState) { + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); + EXPECT_FALSE(machine_->IsWatchHealthy()); + EXPECT_FALSE(machine_->IsReadyForPromotion()); + EXPECT_EQ(0, machine_->GetConsecutiveErrors()); + EXPECT_EQ(0, machine_->GetReconnectCount()); +} + +// ========== Basic State Transition Tests ========== + +TEST_F(StandbyStateMachineTest, TestStartTransition) { + auto result = machine_->ProcessEvent(StandbyEvent::START); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::STOPPED, result.old_state); + EXPECT_EQ(StandbyState::CONNECTING, result.new_state); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + // CONNECTING 状态下还未真正开始同步，因此 IsRunning/IsConnected 都应为 false + EXPECT_FALSE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); +} + +TEST_F(StandbyStateMachineTest, TestConnectedTransition) { + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, result.old_state); + EXPECT_EQ(StandbyState::SYNCING, result.new_state); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_TRUE(machine_->IsConnected()); +} + +TEST_F(StandbyStateMachineTest, TestSyncCompleteTransition) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::WATCHING, result.new_state); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_TRUE(machine_->IsConnected()); + EXPECT_TRUE(machine_->IsWatchHealthy()); + EXPECT_TRUE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestWatchHealthyNoOp) { + ReachWatchingState(); + + // WATCH_HEALTHY in WATCHING state is a no-op (stays in WATCHING) + auto result = machine_->ProcessEvent(StandbyEvent::WATCH_HEALTHY); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::WATCHING, result.new_state); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestWatchBrokenTransition) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsWatchHealthy()); + EXPECT_FALSE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestDisconnectedFromWatching) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestPromoteTransition) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::PROMOTING, result.new_state); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); + EXPECT_TRUE(machine_->IsConnected()); + EXPECT_FALSE(machine_->IsWatchHealthy()); + EXPECT_FALSE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestPromotionSuccessTransition) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::PROMOTING, result.old_state); + EXPECT_EQ(StandbyState::PROMOTED, result.new_state); + EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); +} + +TEST_F(StandbyStateMachineTest, TestPromotionFailedTransition) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::PROMOTION_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::PROMOTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); +} + +TEST_F(StandbyStateMachineTest, TestStopTransition) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::STOPPED, result.new_state); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); + EXPECT_FALSE(machine_->IsConnected()); +} + +// ========== Error and Failure State Tests ========== + +TEST_F(StandbyStateMachineTest, TestConnectionFailedFromConnecting) { + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::CONNECTION_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + EXPECT_FALSE(machine_->IsRunning()); +} + +TEST_F(StandbyStateMachineTest, TestFatalErrorFromConnecting) { + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestSyncFailedFromSyncing) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::SYNC_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + EXPECT_TRUE(machine_->IsRunning()); +} + +TEST_F(StandbyStateMachineTest, TestDisconnectedFromSyncing) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestFatalErrorFromSyncing) { + ReachSyncingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::SYNCING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestFatalErrorFromWatching) { + ReachWatchingState(); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +// ========== Reconnecting State Tests ========== + +TEST_F(StandbyStateMachineTest, TestReconnectingToSyncing) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECONNECTING, result.old_state); + EXPECT_EQ(StandbyState::SYNCING, result.new_state); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestReconnectingToFailed) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestReconnectingMaxErrors) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECONNECTING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +// ========== Recovering State Tests ========== + +TEST_F(StandbyStateMachineTest, TestRecoveringToWatching) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::RECOVERY_SUCCESS); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::WATCHING, result.new_state); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestRecoveringToReconnecting) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::RECOVERY_FAILED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestRecoveringDisconnected) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::DISCONNECTED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::RECONNECTING, result.new_state); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestRecoveringFatalError) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::RECOVERING, result.old_state); + EXPECT_EQ(StandbyState::FAILED, result.new_state); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); +} + +// ========== Failed State Tests ========== + +TEST_F(StandbyStateMachineTest, TestFailedToStopped) { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::FAILED, result.old_state); + EXPECT_EQ(StandbyState::STOPPED, result.new_state); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestFailedToConnecting) { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::FATAL_ERROR); + EXPECT_EQ(StandbyState::FAILED, machine_->GetState()); + + // Allow restart from FAILED state + auto result = machine_->ProcessEvent(StandbyEvent::START); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::FAILED, result.old_state); + EXPECT_EQ(StandbyState::CONNECTING, result.new_state); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); +} + +// ========== Promoted State Tests ========== + +TEST_F(StandbyStateMachineTest, TestPromotedToStopped) { + ReachWatchingState(); + machine_->ProcessEvent(StandbyEvent::PROMOTE); + machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState()); + + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::PROMOTED, result.old_state); + EXPECT_EQ(StandbyState::STOPPED, result.new_state); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +// ========== Invalid Transition Tests ========== + +TEST_F(StandbyStateMachineTest, TestInvalidTransitions) { + // Cannot transition from STOPPED directly to WATCHING + auto result1 = machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_FALSE(result1.allowed); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + + // Cannot promote when not in WATCHING state + ReachSyncingState(); + auto result2 = machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_FALSE(result2.allowed); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + + // Cannot transition from STOPPED to CONNECTED + machine_->ProcessEvent(StandbyEvent::STOP); + auto result3 = machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_FALSE(result3.allowed); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +// ========== Error Handling Tests ========== + +TEST_F(StandbyStateMachineTest, TestConsecutiveErrors) { + ReachWatchingState(); + + // Simulate multiple errors + for (int i = 0; i < 5; ++i) { + machine_->IncrementErrors(); + } + EXPECT_EQ(5, machine_->GetConsecutiveErrors()); + + // Reset errors + machine_->ResetErrors(); + EXPECT_EQ(0, machine_->GetConsecutiveErrors()); +} + +TEST_F(StandbyStateMachineTest, TestMaxErrorsReachedAutoTransition) { + ReachWatchingState(); + + // IncrementErrors() automatically triggers MAX_ERRORS_REACHED when threshold is reached + for (int i = 0; i < StandbyStateMachine::kMaxConsecutiveErrors; ++i) { + machine_->IncrementErrors(); + } + + // Should have transitioned to RECOVERING (from WATCHING on MAX_ERRORS_REACHED) + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + EXPECT_EQ(StandbyStateMachine::kMaxConsecutiveErrors, + machine_->GetConsecutiveErrors()); +} + +TEST_F(StandbyStateMachineTest, TestMaxErrorsReachedManual) { + ReachWatchingState(); + + // Manually trigger MAX_ERRORS_REACHED + auto result = machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::WATCHING, result.old_state); + EXPECT_EQ(StandbyState::RECOVERING, result.new_state); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestReconnectCount) { + EXPECT_EQ(0, machine_->GetReconnectCount()); + + machine_->IncrementReconnectCount(); + EXPECT_EQ(1, machine_->GetReconnectCount()); + + machine_->IncrementReconnectCount(); + EXPECT_EQ(2, machine_->GetReconnectCount()); + + machine_->ResetReconnectCount(); + EXPECT_EQ(0, machine_->GetReconnectCount()); +} + +// ========== Callback Tests ========== + +TEST_F(StandbyStateMachineTest, TestStateChangeCallback) { + std::vector state_history; + std::vector event_history; + + machine_->RegisterCallback( + [&](StandbyState old_state, StandbyState new_state, StandbyEvent event) { + state_history.push_back(new_state); + event_history.push_back(event); + }); + + // Trigger state transitions + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + + // Verify callbacks were called + EXPECT_EQ(3, state_history.size()); + EXPECT_EQ(StandbyState::CONNECTING, state_history[0]); + EXPECT_EQ(StandbyState::SYNCING, state_history[1]); + EXPECT_EQ(StandbyState::WATCHING, state_history[2]); + EXPECT_EQ(StandbyEvent::START, event_history[0]); + EXPECT_EQ(StandbyEvent::CONNECTED, event_history[1]); + EXPECT_EQ(StandbyEvent::SYNC_COMPLETE, event_history[2]); +} + +TEST_F(StandbyStateMachineTest, TestMultipleCallbacks) { + int callback1_count = 0; + int callback2_count = 0; + + machine_->RegisterCallback([&](StandbyState, StandbyState, StandbyEvent) { + callback1_count++; + }); + machine_->RegisterCallback([&](StandbyState, StandbyState, StandbyEvent) { + callback2_count++; + }); + + // Trigger state transitions + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + + // Both callbacks should be called + EXPECT_EQ(2, callback1_count); + EXPECT_EQ(2, callback2_count); +} + +TEST_F(StandbyStateMachineTest, TestCallbackExceptionHandling) { + bool callback_called = false; + + machine_->RegisterCallback([&](StandbyState, StandbyState, StandbyEvent) { + callback_called = true; + }); + + // Callback 被调用且不影响状态转换 + auto result = machine_->ProcessEvent(StandbyEvent::START); + EXPECT_TRUE(result.allowed); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + EXPECT_TRUE(callback_called); +} + +// ========== History Tests ========== + +TEST_F(StandbyStateMachineTest, TestTransitionHistory) { + // Perform several transitions + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::CONNECTED); + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + + auto history = machine_->GetTransitionHistory(10); + EXPECT_EQ(3, history.size()); + EXPECT_EQ(StandbyState::STOPPED, history[0].from_state); + EXPECT_EQ(StandbyState::CONNECTING, history[0].to_state); + EXPECT_EQ(StandbyEvent::START, history[0].event); + + EXPECT_EQ(StandbyState::CONNECTING, history[1].from_state); + EXPECT_EQ(StandbyState::SYNCING, history[1].to_state); + EXPECT_EQ(StandbyEvent::CONNECTED, history[1].event); + + EXPECT_EQ(StandbyState::SYNCING, history[2].from_state); + EXPECT_EQ(StandbyState::WATCHING, history[2].to_state); + EXPECT_EQ(StandbyEvent::SYNC_COMPLETE, history[2].event); +} + +TEST_F(StandbyStateMachineTest, TestTransitionHistoryLimit) { + // Perform many transitions to test history limit + for (int i = 0; i < 20; ++i) { + machine_->ProcessEvent(StandbyEvent::START); + machine_->ProcessEvent(StandbyEvent::STOP); + } + + // Request limited history + auto history = machine_->GetTransitionHistory(5); + EXPECT_LE(history.size(), 5); +} + +TEST_F(StandbyStateMachineTest, TestTimeInState) { + machine_->ProcessEvent(StandbyEvent::START); + + // Wait a bit + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + auto time_in_state = machine_->GetTimeInCurrentState(); + EXPECT_GE(time_in_state.count(), 100); + EXPECT_LE(time_in_state.count(), 200); // Allow some margin for test execution +} + +// ========== Concurrent Tests ========== + +TEST_F(StandbyStateMachineTest, TestConcurrentStateQueries) { + ReachWatchingState(); + + // Multiple threads querying state concurrently + std::vector threads; + std::atomic success_count{0}; + + for (int i = 0; i < 10; ++i) { + threads.emplace_back([&]() { + for (int j = 0; j < 100; ++j) { + StandbyState state = machine_->GetState(); + if (state == StandbyState::WATCHING) { + success_count++; + } + } + }); + } + + for (auto& t : threads) { + t.join(); + } + + EXPECT_EQ(1000, success_count.load()); +} + +TEST_F(StandbyStateMachineTest, TestConcurrentEventProcessing) { + ReachWatchingState(); + + // Multiple threads trying to process events concurrently + // Only one should succeed (state machine should serialize) + std::vector threads; + std::atomic success_count{0}; + std::atomic failure_count{0}; + + for (int i = 0; i < 10; ++i) { + threads.emplace_back([&]() { + auto result = machine_->ProcessEvent(StandbyEvent::STOP); + if (result.allowed) { + success_count++; + } else { + failure_count++; + } + }); + } + + for (auto& t : threads) { + t.join(); + } + + // Only one STOP should succeed (transition to STOPPED) + EXPECT_EQ(1, success_count.load()); + EXPECT_EQ(9, failure_count.load()); + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); +} + +// ========== State Query Tests ========== + +TEST_F(StandbyStateMachineTest, TestIsRunning) { + EXPECT_FALSE(machine_->IsRunning()); // STOPPED + + machine_->ProcessEvent(StandbyEvent::START); + // CONNECTING 仅表示正在建立连接，还未开始同步，不视为 running + EXPECT_FALSE(machine_->IsRunning()); // CONNECTING + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(machine_->IsRunning()); // SYNCING + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_TRUE(machine_->IsRunning()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_FALSE(machine_->IsRunning()); // STOPPED +} + +TEST_F(StandbyStateMachineTest, TestIsConnected) { + EXPECT_FALSE(machine_->IsConnected()); // STOPPED + + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_FALSE(machine_->IsConnected()); // CONNECTING + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_TRUE(machine_->IsConnected()); // SYNCING + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_TRUE(machine_->IsConnected()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::STOP); + EXPECT_FALSE(machine_->IsConnected()); // STOPPED +} + +TEST_F(StandbyStateMachineTest, TestIsWatchHealthy) { + EXPECT_FALSE(machine_->IsWatchHealthy()); // STOPPED + + ReachWatchingState(); + EXPECT_TRUE(machine_->IsWatchHealthy()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_FALSE(machine_->IsWatchHealthy()); // RECONNECTING +} + +TEST_F(StandbyStateMachineTest, TestIsReadyForPromotion) { + EXPECT_FALSE(machine_->IsReadyForPromotion()); // STOPPED + + ReachWatchingState(); + EXPECT_TRUE(machine_->IsReadyForPromotion()); // WATCHING + + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_FALSE(machine_->IsReadyForPromotion()); // PROMOTING +} + +// ========== Complete State Machine Flow Tests ========== + +TEST_F(StandbyStateMachineTest, TestCompleteNormalFlow) { + // Complete flow: STOPPED -> CONNECTING -> SYNCING -> WATCHING + EXPECT_EQ(StandbyState::STOPPED, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::START); + EXPECT_EQ(StandbyState::CONNECTING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); + EXPECT_TRUE(machine_->IsReadyForPromotion()); +} + +TEST_F(StandbyStateMachineTest, TestCompletePromotionFlow) { + // Complete promotion flow + ReachWatchingState(); + + machine_->ProcessEvent(StandbyEvent::PROMOTE); + EXPECT_EQ(StandbyState::PROMOTING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + EXPECT_EQ(StandbyState::PROMOTED, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestCompleteReconnectFlow) { + // Complete reconnect flow: WATCHING -> RECONNECTING -> SYNCING -> WATCHING + ReachWatchingState(); + + machine_->ProcessEvent(StandbyEvent::WATCH_BROKEN); + EXPECT_EQ(StandbyState::RECONNECTING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::CONNECTED); + EXPECT_EQ(StandbyState::SYNCING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::SYNC_COMPLETE); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +TEST_F(StandbyStateMachineTest, TestCompleteRecoveryFlow) { + // Complete recovery flow: WATCHING -> RECOVERING -> WATCHING + ReachWatchingState(); + + machine_->ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + EXPECT_EQ(StandbyState::RECOVERING, machine_->GetState()); + + machine_->ProcessEvent(StandbyEvent::RECOVERY_SUCCESS); + EXPECT_EQ(StandbyState::WATCHING, machine_->GetState()); +} + +} // namespace mooncake::test + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} +