From 60c50f53c62b194237048e23ef8252afb9d3aa66 Mon Sep 17 00:00:00 2001 From: zhangyupeng Date: Fri, 13 Nov 2026 21:31:13 +0800 Subject: [PATCH 1/4] transfer: wait for inflight batches before unregisterLocalMemory Mid-flight ibv_dereg_mr causes IBV_WC_LOC_PROT_ERR storms that trip the RDMA context breaker and leave Device-not-active sticky until TE restart (Type-2 sticky). Enforce the documented contract by waiting for live/deferred batches that still reference the buffer, returning ERR_BATCH_BUSY on timeout. Fixes #5481 Signed-off-by: zhangyupeng --- .../include/multi_transport.h | 18 +++++ .../src/multi_transport.cpp | 77 +++++++++++++++++++ .../src/transfer_engine_impl.cpp | 30 ++++++++ 3 files changed, 125 insertions(+) diff --git a/mooncake-transfer-engine/include/multi_transport.h b/mooncake-transfer-engine/include/multi_transport.h index d6ca3e84dfb..eb955ba0b23 100644 --- a/mooncake-transfer-engine/include/multi_transport.h +++ b/mooncake-transfer-engine/include/multi_transport.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "transport/transport.h" #include "task_congestion_status.h" @@ -105,6 +106,13 @@ class MultiTransport { std::vector listTransports(); + // Block until no live/deferred batch still references [addr, + // addr+length). Used by unregisterLocalMemory so ibv_dereg_mr + // cannot race outstanding RDMA WRs (local protection → sticky + // Device-not-active). length==0 matches exact source pointer. + // Returns BatchBusy if timeout_ms elapses while still busy. + Status waitMemoryIdle(void *addr, size_t length, int64_t timeout_ms); + void *getBaseAddr(); private: @@ -148,6 +156,12 @@ class MultiTransport { void deferredCleanupLoop(); + void trackLiveBatch(BatchID batch_id); + void untrackLiveBatch(BatchID batch_id); + bool memoryReferencedByBatch(BatchID batch_id, void *addr, + size_t length) const; + bool memoryBusyLocked(void *addr, size_t length) const; + #ifdef ENABLE_MULTI_PROTOCOL Status mp_selectTransport(const TransferRequest &entry, Transport *&transport, @@ -162,6 +176,10 @@ class MultiTransport { std::unordered_map> batch_desc_set_; std::mutex deferred_cleanup_mutex_; + // Always-on live batch index for unregister quiesce (#5481). + // Distinct from optional CONFIG_USE_BATCH_DESC_SET ownership map. + mutable std::mutex live_batches_mutex_; + std::unordered_set live_batches_; std::condition_variable deferred_cleanup_cv_; std::unordered_map> deferred_cleanup_batches_; diff --git a/mooncake-transfer-engine/src/multi_transport.cpp b/mooncake-transfer-engine/src/multi_transport.cpp index 27cac28ea17..1f7169de15f 100644 --- a/mooncake-transfer-engine/src/multi_transport.cpp +++ b/mooncake-transfer-engine/src/multi_transport.cpp @@ -116,6 +116,7 @@ MultiTransport::BatchID MultiTransport::allocateBatchID(size_t batch_size) { batch_desc_set_[batch_desc->id] = batch_desc; batch_desc_lock_.unlock(); #endif + trackLiveBatch(batch_desc->id); return batch_desc->id; } @@ -202,6 +203,7 @@ Status MultiTransport::tryFreeBatchID( if (before_delete) { before_delete(); } + untrackLiveBatch(batch_id); delete &batch_desc; #ifdef CONFIG_USE_BATCH_DESC_SET RWSpinlock::WriteGuard guard(batch_desc_lock_); @@ -1208,3 +1210,78 @@ void* MultiTransport::getBaseAddr() { } } // namespace mooncake + +void MultiTransport::trackLiveBatch(BatchID batch_id) { + std::lock_guard guard(live_batches_mutex_); + live_batches_.insert(batch_id); +} + +void MultiTransport::untrackLiveBatch(BatchID batch_id) { + std::lock_guard guard(live_batches_mutex_); + live_batches_.erase(batch_id); +} + +bool MultiTransport::memoryReferencedByBatch(BatchID batch_id, void* addr, + size_t length) const { + if (!batch_id || batch_id == (BatchID)ERR_MEMORY) return false; + auto& batch_desc = *((BatchDesc*)(batch_id)); + const uintptr_t a0 = reinterpret_cast(addr); + const uintptr_t a1 = length ? (a0 + length) : (a0 + 1); + for (auto& task : batch_desc.task_list) { + if (__atomic_load_n(&task.is_finished, __ATOMIC_ACQUIRE)) continue; + if (task.request) { + const uintptr_t s = + reinterpret_cast(task.request->source); + const size_t n = task.request->length; + const uintptr_t s1 = s + (n ? n : 1); + if (s < a1 && a0 < s1) return true; + } + for (auto* slice : task.slice_list) { + if (!slice) continue; + const uintptr_t s = reinterpret_cast(slice->source_addr); + const size_t n = slice->length; + const uintptr_t s1 = s + (n ? n : 1); + if (s < a1 && a0 < s1) return true; + } + } + return false; +} + +bool MultiTransport::memoryBusyLocked(void* addr, size_t length) const { + for (BatchID batch_id : live_batches_) { + if (memoryReferencedByBatch(batch_id, addr, length)) return true; + } + return false; +} + +Status MultiTransport::waitMemoryIdle(void* addr, size_t length, + int64_t timeout_ms) { + if (!addr) return Status::InvalidArgument("addr is null"); + const auto deadline = + std::chrono::steady_clock::now() + + std::chrono::milliseconds(std::max(timeout_ms, 0)); + for (;;) { + bool busy = false; + { + std::lock_guard live_guard(live_batches_mutex_); + busy = memoryBusyLocked(addr, length); + } + if (!busy) { + std::lock_guard def_guard(deferred_cleanup_mutex_); + for (const auto& entry : deferred_cleanup_batches_) { + if (memoryReferencedByBatch(entry.first, addr, length)) { + busy = true; + break; + } + } + } + if (!busy) return Status::OK(); + if (std::chrono::steady_clock::now() >= deadline) { + return Status::BatchBusy( + "unregisterLocalMemory blocked: transfers still reference " + "buffer"); + } + deferred_cleanup_cv_.notify_one(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +} diff --git a/mooncake-transfer-engine/src/transfer_engine_impl.cpp b/mooncake-transfer-engine/src/transfer_engine_impl.cpp index 8a42aa57f72..070dd1a1a22 100644 --- a/mooncake-transfer-engine/src/transfer_engine_impl.cpp +++ b/mooncake-transfer-engine/src/transfer_engine_impl.cpp @@ -840,6 +840,24 @@ int TransferEngineImpl::registerLocalMemory(void* addr, size_t length, int TransferEngineImpl::unregisterLocalMemory(void* addr, bool update_metadata) { + // Quiesce in-flight batches that still reference this buffer before + // transport deregister. Mid-flight ibv_dereg_mr causes LOC_PROT storms + // that latch sticky "Device is not active" (#5481). + if (multi_transports_) { + Status st = multi_transports_->waitMemoryIdle(addr, /*length=*/0, + /*timeout_ms=*/5000); + if (st.IsBatchBusy()) { + LOG(ERROR) << "unregisterLocalMemory: buffer still in flight, " + "refusing deregister addr=" + << addr; + return ERR_BATCH_BUSY; + } + if (!st.ok()) { + LOG(WARNING) << "unregisterLocalMemory waitMemoryIdle: " + << st.ToString(); + } + } + // Best-effort: try every transport so one failure can't leave the region // registered on the others; mirrors unregisterLocalMemoryBatch (#2869). int first_error = 0; @@ -1056,6 +1074,18 @@ int TransferEngineImpl::registerLocalMemoryBatch( int TransferEngineImpl::unregisterLocalMemoryBatch( const std::vector& addr_list) { + if (multi_transports_) { + for (auto* addr : addr_list) { + Status st = multi_transports_->waitMemoryIdle(addr, /*length=*/0, + /*timeout_ms=*/5000); + if (st.IsBatchBusy()) { + LOG(ERROR) << "unregisterLocalMemoryBatch: buffer still in " + "flight, refusing deregister addr=" + << addr; + return ERR_BATCH_BUSY; + } + } + } int first_error = 0; for (auto transport : multi_transports_->listTransports()) { int ret = transport->unregisterLocalMemoryBatch(addr_list); From 2768cfcbf7529c1e619829a43a58cdd650532593 Mon Sep 17 00:00:00 2001 From: zhangyupeng Date: Mon, 16 Nov 2026 10:02:34 +0800 Subject: [PATCH 2/4] transfer: keep waitMemoryIdle helpers inside mooncake namespace CI builds failed because trackLiveBatch/waitMemoryIdle were appended after the namespace mooncake closing brace, so MultiTransport/BatchID/ Status were undeclared. Signed-off-by: zhangyupeng Co-authored-by: Cursor --- mooncake-transfer-engine/src/multi_transport.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mooncake-transfer-engine/src/multi_transport.cpp b/mooncake-transfer-engine/src/multi_transport.cpp index 1f7169de15f..efe0a0c8a22 100644 --- a/mooncake-transfer-engine/src/multi_transport.cpp +++ b/mooncake-transfer-engine/src/multi_transport.cpp @@ -1209,7 +1209,7 @@ void* MultiTransport::getBaseAddr() { return 0; } -} // namespace mooncake + void MultiTransport::trackLiveBatch(BatchID batch_id) { std::lock_guard guard(live_batches_mutex_); @@ -1285,3 +1285,5 @@ Status MultiTransport::waitMemoryIdle(void* addr, size_t length, std::this_thread::sleep_for(std::chrono::milliseconds(1)); } } + +} // namespace mooncake From 8ea042ddb2a462a07679769d2ad9c0096aaa2856 Mon Sep 17 00:00:00 2001 From: zhangyupeng Date: Mon, 16 Nov 2026 10:09:20 +0800 Subject: [PATCH 3/4] transfer: drop extra blank lines for clang-format Signed-off-by: zhangyupeng Co-authored-by: Cursor --- mooncake-transfer-engine/src/multi_transport.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/mooncake-transfer-engine/src/multi_transport.cpp b/mooncake-transfer-engine/src/multi_transport.cpp index efe0a0c8a22..c987c678b58 100644 --- a/mooncake-transfer-engine/src/multi_transport.cpp +++ b/mooncake-transfer-engine/src/multi_transport.cpp @@ -1209,8 +1209,6 @@ void* MultiTransport::getBaseAddr() { return 0; } - - void MultiTransport::trackLiveBatch(BatchID batch_id) { std::lock_guard guard(live_batches_mutex_); live_batches_.insert(batch_id); From 7113f56bd6551ff25e530d29518397a3efe800bc Mon Sep 17 00:00:00 2001 From: zhangyupeng Date: Mon, 16 Nov 2026 11:14:15 +0800 Subject: [PATCH 4/4] transfer: shard the live-batch index used by unregister quiesce Concurrent allocate/free now contend on one of 64 stripes instead of a process-wide mutex. waitMemoryIdle still scans every stripe and confirms a clean result under all shard locks. Signed-off-by: zhangyupeng Co-authored-by: Cursor --- .../include/multi_transport.h | 18 +++++++-- .../src/multi_transport.cpp | 37 +++++++++++++------ 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/mooncake-transfer-engine/include/multi_transport.h b/mooncake-transfer-engine/include/multi_transport.h index a1c0f431948..3c30f8325dc 100644 --- a/mooncake-transfer-engine/include/multi_transport.h +++ b/mooncake-transfer-engine/include/multi_transport.h @@ -15,6 +15,7 @@ #ifndef MULTI_TRANSPORT_H_ #define MULTI_TRANSPORT_H_ +#include #include #include #include @@ -179,7 +180,9 @@ class MultiTransport { void untrackLiveBatch(BatchID batch_id); bool memoryReferencedByBatch(BatchID batch_id, void *addr, size_t length) const; - bool memoryBusyLocked(void *addr, size_t length) const; + // Scans live batches. A clean result is confirmed under every shard + // lock so a batch inserted into an already-scanned shard is not missed. + bool memoryBusy(void *addr, size_t length) const; // Stop a future or already-published event record from being delivered. // Does not delete the batch. @@ -205,8 +208,17 @@ class MultiTransport { std::mutex deferred_cleanup_mutex_; // Always-on live batch index for unregister quiesce (#5481). // Distinct from optional CONFIG_USE_BATCH_DESC_SET ownership map. - mutable std::mutex live_batches_mutex_; - std::unordered_set live_batches_; + // Sharded so concurrent allocate/free only contend on one stripe; + // waitMemoryIdle (unregister) is the only full scan. + static constexpr size_t kLiveBatchShards = 64; + struct LiveBatchShard { + mutable std::mutex mu; + std::unordered_set batches; + }; + std::array live_batch_shards_{}; + static size_t liveBatchShard(BatchID batch_id) { + return static_cast(batch_id % kLiveBatchShards); + } std::condition_variable deferred_cleanup_cv_; std::unordered_map> deferred_cleanup_batches_; diff --git a/mooncake-transfer-engine/src/multi_transport.cpp b/mooncake-transfer-engine/src/multi_transport.cpp index d367114cb87..b96e9d15979 100644 --- a/mooncake-transfer-engine/src/multi_transport.cpp +++ b/mooncake-transfer-engine/src/multi_transport.cpp @@ -1476,13 +1476,15 @@ void* MultiTransport::getBaseAddr() { } void MultiTransport::trackLiveBatch(BatchID batch_id) { - std::lock_guard guard(live_batches_mutex_); - live_batches_.insert(batch_id); + auto& shard = live_batch_shards_[liveBatchShard(batch_id)]; + std::lock_guard guard(shard.mu); + shard.batches.insert(batch_id); } void MultiTransport::untrackLiveBatch(BatchID batch_id) { - std::lock_guard guard(live_batches_mutex_); - live_batches_.erase(batch_id); + auto& shard = live_batch_shards_[liveBatchShard(batch_id)]; + std::lock_guard guard(shard.mu); + shard.batches.erase(batch_id); } bool MultiTransport::memoryReferencedByBatch(BatchID batch_id, void* addr, @@ -1511,9 +1513,25 @@ bool MultiTransport::memoryReferencedByBatch(BatchID batch_id, void* addr, return false; } -bool MultiTransport::memoryBusyLocked(void* addr, size_t length) const { - for (BatchID batch_id : live_batches_) { - if (memoryReferencedByBatch(batch_id, addr, length)) return true; +bool MultiTransport::memoryBusy(void* addr, size_t length) const { + auto shard_hits = [&](const LiveBatchShard& shard) { + for (BatchID batch_id : shard.batches) { + if (memoryReferencedByBatch(batch_id, addr, length)) return true; + } + return false; + }; + for (const auto& shard : live_batch_shards_) { + std::lock_guard guard(shard.mu); + if (shard_hits(shard)) return true; + } + // A batch can land in a shard we already released. Confirm once while + // holding every shard in index order (same order as any other full scan). + std::unique_lock locks[kLiveBatchShards]; + for (size_t i = 0; i < kLiveBatchShards; ++i) { + locks[i] = std::unique_lock(live_batch_shards_[i].mu); + } + for (const auto& shard : live_batch_shards_) { + if (shard_hits(shard)) return true; } return false; } @@ -1526,10 +1544,7 @@ Status MultiTransport::waitMemoryIdle(void* addr, size_t length, std::chrono::milliseconds(std::max(timeout_ms, 0)); for (;;) { bool busy = false; - { - std::lock_guard live_guard(live_batches_mutex_); - busy = memoryBusyLocked(addr, length); - } + busy = memoryBusy(addr, length); if (!busy) { std::lock_guard def_guard(deferred_cleanup_mutex_); for (const auto& entry : deferred_cleanup_batches_) {