diff --git a/mooncake-transfer-engine/include/multi_transport.h b/mooncake-transfer-engine/include/multi_transport.h index 0b9939ba1b5..3c30f8325dc 100644 --- a/mooncake-transfer-engine/include/multi_transport.h +++ b/mooncake-transfer-engine/include/multi_transport.h @@ -15,6 +15,7 @@ #ifndef MULTI_TRANSPORT_H_ #define MULTI_TRANSPORT_H_ +#include #include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include "transport/transport.h" @@ -124,6 +126,13 @@ class MultiTransport { std::vector listTransports(); + // Block until no live/deferred batch still references [addr, + // addr+length). Used by unregisterLocalMemory so ibv_dereg_mr + // cannot race outstanding RDMA WRs (local protection → sticky + // Device-not-active). length==0 matches exact source pointer. + // Returns BatchBusy if timeout_ms elapses while still busy. + Status waitMemoryIdle(void *addr, size_t length, int64_t timeout_ms); + void *getBaseAddr(); private: @@ -167,6 +176,14 @@ class MultiTransport { void deferredCleanupLoop(); + void trackLiveBatch(BatchID batch_id); + void untrackLiveBatch(BatchID batch_id); + bool memoryReferencedByBatch(BatchID batch_id, void *addr, + size_t length) const; + // Scans live batches. A clean result is confirmed under every shard + // lock so a batch inserted into an already-scanned shard is not missed. + bool memoryBusy(void *addr, size_t length) const; + // Stop a future or already-published event record from being delivered. // Does not delete the batch. void suppressCompletion(BatchDesc *batch); @@ -189,6 +206,19 @@ class MultiTransport { std::unordered_map> batch_desc_set_; std::mutex deferred_cleanup_mutex_; + // Always-on live batch index for unregister quiesce (#5481). + // Distinct from optional CONFIG_USE_BATCH_DESC_SET ownership map. + // Sharded so concurrent allocate/free only contend on one stripe; + // waitMemoryIdle (unregister) is the only full scan. + static constexpr size_t kLiveBatchShards = 64; + struct LiveBatchShard { + mutable std::mutex mu; + std::unordered_set batches; + }; + std::array live_batch_shards_{}; + static size_t liveBatchShard(BatchID batch_id) { + return static_cast(batch_id % kLiveBatchShards); + } std::condition_variable deferred_cleanup_cv_; std::unordered_map> deferred_cleanup_batches_; diff --git a/mooncake-transfer-engine/src/multi_transport.cpp b/mooncake-transfer-engine/src/multi_transport.cpp index 4d24be69a02..b96e9d15979 100644 --- a/mooncake-transfer-engine/src/multi_transport.cpp +++ b/mooncake-transfer-engine/src/multi_transport.cpp @@ -160,6 +160,7 @@ MultiTransport::BatchID MultiTransport::allocateBatchID(size_t batch_size) { batch_desc_set_[batch_desc->id] = batch_desc; batch_desc_lock_.unlock(); #endif + trackLiveBatch(batch_desc->id); return batch_desc->id; } @@ -246,6 +247,7 @@ Status MultiTransport::tryFreeBatchID( "BatchID cannot be freed until completion callbacks are " "quiescent"); } + untrackLiveBatch(batch_id); // The queue holds its own nodes, so the descriptor can be deleted once // no callback is still inside it. suppressCompletion keeps a published // id out of a later drain. @@ -1473,4 +1475,94 @@ void* MultiTransport::getBaseAddr() { return 0; } +void MultiTransport::trackLiveBatch(BatchID batch_id) { + auto& shard = live_batch_shards_[liveBatchShard(batch_id)]; + std::lock_guard guard(shard.mu); + shard.batches.insert(batch_id); +} + +void MultiTransport::untrackLiveBatch(BatchID batch_id) { + auto& shard = live_batch_shards_[liveBatchShard(batch_id)]; + std::lock_guard guard(shard.mu); + shard.batches.erase(batch_id); +} + +bool MultiTransport::memoryReferencedByBatch(BatchID batch_id, void* addr, + size_t length) const { + if (!batch_id || batch_id == (BatchID)ERR_MEMORY) return false; + auto& batch_desc = *((BatchDesc*)(batch_id)); + const uintptr_t a0 = reinterpret_cast(addr); + const uintptr_t a1 = length ? (a0 + length) : (a0 + 1); + for (auto& task : batch_desc.task_list) { + if (__atomic_load_n(&task.is_finished, __ATOMIC_ACQUIRE)) continue; + if (task.request) { + const uintptr_t s = + reinterpret_cast(task.request->source); + const size_t n = task.request->length; + const uintptr_t s1 = s + (n ? n : 1); + if (s < a1 && a0 < s1) return true; + } + for (auto* slice : task.slice_list) { + if (!slice) continue; + const uintptr_t s = reinterpret_cast(slice->source_addr); + const size_t n = slice->length; + const uintptr_t s1 = s + (n ? n : 1); + if (s < a1 && a0 < s1) return true; + } + } + return false; +} + +bool MultiTransport::memoryBusy(void* addr, size_t length) const { + auto shard_hits = [&](const LiveBatchShard& shard) { + for (BatchID batch_id : shard.batches) { + if (memoryReferencedByBatch(batch_id, addr, length)) return true; + } + return false; + }; + for (const auto& shard : live_batch_shards_) { + std::lock_guard guard(shard.mu); + if (shard_hits(shard)) return true; + } + // A batch can land in a shard we already released. Confirm once while + // holding every shard in index order (same order as any other full scan). + std::unique_lock locks[kLiveBatchShards]; + for (size_t i = 0; i < kLiveBatchShards; ++i) { + locks[i] = std::unique_lock(live_batch_shards_[i].mu); + } + for (const auto& shard : live_batch_shards_) { + if (shard_hits(shard)) return true; + } + return false; +} + +Status MultiTransport::waitMemoryIdle(void* addr, size_t length, + int64_t timeout_ms) { + if (!addr) return Status::InvalidArgument("addr is null"); + const auto deadline = + std::chrono::steady_clock::now() + + std::chrono::milliseconds(std::max(timeout_ms, 0)); + for (;;) { + bool busy = false; + busy = memoryBusy(addr, length); + if (!busy) { + std::lock_guard def_guard(deferred_cleanup_mutex_); + for (const auto& entry : deferred_cleanup_batches_) { + if (memoryReferencedByBatch(entry.first, addr, length)) { + busy = true; + break; + } + } + } + if (!busy) return Status::OK(); + if (std::chrono::steady_clock::now() >= deadline) { + return Status::BatchBusy( + "unregisterLocalMemory blocked: transfers still reference " + "buffer"); + } + deferred_cleanup_cv_.notify_one(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +} + } // namespace mooncake diff --git a/mooncake-transfer-engine/src/transfer_engine_impl.cpp b/mooncake-transfer-engine/src/transfer_engine_impl.cpp index 8a42aa57f72..070dd1a1a22 100644 --- a/mooncake-transfer-engine/src/transfer_engine_impl.cpp +++ b/mooncake-transfer-engine/src/transfer_engine_impl.cpp @@ -840,6 +840,24 @@ int TransferEngineImpl::registerLocalMemory(void* addr, size_t length, int TransferEngineImpl::unregisterLocalMemory(void* addr, bool update_metadata) { + // Quiesce in-flight batches that still reference this buffer before + // transport deregister. Mid-flight ibv_dereg_mr causes LOC_PROT storms + // that latch sticky "Device is not active" (#5481). + if (multi_transports_) { + Status st = multi_transports_->waitMemoryIdle(addr, /*length=*/0, + /*timeout_ms=*/5000); + if (st.IsBatchBusy()) { + LOG(ERROR) << "unregisterLocalMemory: buffer still in flight, " + "refusing deregister addr=" + << addr; + return ERR_BATCH_BUSY; + } + if (!st.ok()) { + LOG(WARNING) << "unregisterLocalMemory waitMemoryIdle: " + << st.ToString(); + } + } + // Best-effort: try every transport so one failure can't leave the region // registered on the others; mirrors unregisterLocalMemoryBatch (#2869). int first_error = 0; @@ -1056,6 +1074,18 @@ int TransferEngineImpl::registerLocalMemoryBatch( int TransferEngineImpl::unregisterLocalMemoryBatch( const std::vector& addr_list) { + if (multi_transports_) { + for (auto* addr : addr_list) { + Status st = multi_transports_->waitMemoryIdle(addr, /*length=*/0, + /*timeout_ms=*/5000); + if (st.IsBatchBusy()) { + LOG(ERROR) << "unregisterLocalMemoryBatch: buffer still in " + "flight, refusing deregister addr=" + << addr; + return ERR_BATCH_BUSY; + } + } + } int first_error = 0; for (auto transport : multi_transports_->listTransports()) { int ret = transport->unregisterLocalMemoryBatch(addr_list);