Repository navigation
transfer: wait for inflight batches before unregisterLocalMemory #5482
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
60c50f5
2768cfc
8ea042d
951e9d1
7113f56
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -160,6 +160,7 @@ MultiTransport::BatchID MultiTransport::allocateBatchID(size_t batch_size) { | |
| batch_desc_set_[batch_desc->id] = batch_desc; | ||
| batch_desc_lock_.unlock(); | ||
| #endif | ||
| trackLiveBatch(batch_desc->id); | ||
| return batch_desc->id; | ||
| } | ||
|
|
||
|
|
@@ -246,6 +247,7 @@ Status MultiTransport::tryFreeBatchID( | |
| "BatchID cannot be freed until completion callbacks are " | ||
| "quiescent"); | ||
| } | ||
| untrackLiveBatch(batch_id); | ||
| // The queue holds its own nodes, so the descriptor can be deleted once | ||
| // no callback is still inside it. suppressCompletion keeps a published | ||
| // id out of a later drain. | ||
|
|
@@ -1473,4 +1475,94 @@ void* MultiTransport::getBaseAddr() { | |
| return 0; | ||
| } | ||
|
|
||
| void MultiTransport::trackLiveBatch(BatchID batch_id) { | ||
| auto& shard = live_batch_shards_[liveBatchShard(batch_id)]; | ||
| std::lock_guard<std::mutex> guard(shard.mu); | ||
| shard.batches.insert(batch_id); | ||
| } | ||
|
|
||
| void MultiTransport::untrackLiveBatch(BatchID batch_id) { | ||
| auto& shard = live_batch_shards_[liveBatchShard(batch_id)]; | ||
| std::lock_guard<std::mutex> guard(shard.mu); | ||
| shard.batches.erase(batch_id); | ||
| } | ||
|
|
||
| bool MultiTransport::memoryReferencedByBatch(BatchID batch_id, void* addr, | ||
| size_t length) const { | ||
| if (!batch_id || batch_id == (BatchID)ERR_MEMORY) return false; | ||
| auto& batch_desc = *((BatchDesc*)(batch_id)); | ||
| const uintptr_t a0 = reinterpret_cast<uintptr_t>(addr); | ||
| const uintptr_t a1 = length ? (a0 + length) : (a0 + 1); | ||
| for (auto& task : batch_desc.task_list) { | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P1] Synchronize reference scanning with task and slice publication The live-batch shard mutex protects set membership and descriptor deletion, but |
||
| if (__atomic_load_n(&task.is_finished, __ATOMIC_ACQUIRE)) continue; | ||
| if (task.request) { | ||
| const uintptr_t s = | ||
| reinterpret_cast<uintptr_t>(task.request->source); | ||
| const size_t n = task.request->length; | ||
|
Comment on lines
+1498
to
+1501
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P1] Do not dereference a borrowed request after submission
|
||
| const uintptr_t s1 = s + (n ? n : 1); | ||
| if (s < a1 && a0 < s1) return true; | ||
| } | ||
| for (auto* slice : task.slice_list) { | ||
| if (!slice) continue; | ||
| const uintptr_t s = reinterpret_cast<uintptr_t>(slice->source_addr); | ||
| const size_t n = slice->length; | ||
| const uintptr_t s1 = s + (n ? n : 1); | ||
| if (s < a1 && a0 < s1) return true; | ||
| } | ||
| } | ||
| return false; | ||
| } | ||
|
|
||
| bool MultiTransport::memoryBusy(void* addr, size_t length) const { | ||
| auto shard_hits = [&](const LiveBatchShard& shard) { | ||
| for (BatchID batch_id : shard.batches) { | ||
| if (memoryReferencedByBatch(batch_id, addr, length)) return true; | ||
| } | ||
| return false; | ||
| }; | ||
| for (const auto& shard : live_batch_shards_) { | ||
| std::lock_guard<std::mutex> guard(shard.mu); | ||
| if (shard_hits(shard)) return true; | ||
| } | ||
| // A batch can land in a shard we already released. Confirm once while | ||
| // holding every shard in index order (same order as any other full scan). | ||
| std::unique_lock<std::mutex> locks[kLiveBatchShards]; | ||
| for (size_t i = 0; i < kLiveBatchShards; ++i) { | ||
| locks[i] = std::unique_lock<std::mutex>(live_batch_shards_[i].mu); | ||
| } | ||
| for (const auto& shard : live_batch_shards_) { | ||
| if (shard_hits(shard)) return true; | ||
| } | ||
| return false; | ||
| } | ||
|
|
||
| Status MultiTransport::waitMemoryIdle(void* addr, size_t length, | ||
| int64_t timeout_ms) { | ||
| if (!addr) return Status::InvalidArgument("addr is null"); | ||
| const auto deadline = | ||
| std::chrono::steady_clock::now() + | ||
| std::chrono::milliseconds(std::max<int64_t>(timeout_ms, 0)); | ||
| for (;;) { | ||
| bool busy = false; | ||
| busy = memoryBusy(addr, length); | ||
| if (!busy) { | ||
| std::lock_guard<std::mutex> def_guard(deferred_cleanup_mutex_); | ||
| for (const auto& entry : deferred_cleanup_batches_) { | ||
| if (memoryReferencedByBatch(entry.first, addr, length)) { | ||
| busy = true; | ||
| break; | ||
| } | ||
| } | ||
| } | ||
| if (!busy) return Status::OK(); | ||
| if (std::chrono::steady_clock::now() >= deadline) { | ||
| return Status::BatchBusy( | ||
| "unregisterLocalMemory blocked: transfers still reference " | ||
| "buffer"); | ||
| } | ||
| deferred_cleanup_cv_.notify_one(); | ||
| std::this_thread::sleep_for(std::chrono::milliseconds(1)); | ||
| } | ||
| } | ||
|
|
||
| } // namespace mooncake | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -840,6 +840,24 @@ int TransferEngineImpl::registerLocalMemory(void* addr, size_t length, | |
|
|
||
| int TransferEngineImpl::unregisterLocalMemory(void* addr, | ||
| bool update_metadata) { | ||
| // Quiesce in-flight batches that still reference this buffer before | ||
| // transport deregister. Mid-flight ibv_dereg_mr causes LOC_PROT storms | ||
| // that latch sticky "Device is not active" (#5481). | ||
| if (multi_transports_) { | ||
| Status st = multi_transports_->waitMemoryIdle(addr, /*length=*/0, | ||
| /*timeout_ms=*/5000); | ||
|
Comment on lines
+847
to
+848
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P1] Check the full registered region before deregistering Passing |
||
| if (st.IsBatchBusy()) { | ||
| LOG(ERROR) << "unregisterLocalMemory: buffer still in flight, " | ||
| "refusing deregister addr=" | ||
| << addr; | ||
| return ERR_BATCH_BUSY; | ||
| } | ||
| if (!st.ok()) { | ||
| LOG(WARNING) << "unregisterLocalMemory waitMemoryIdle: " | ||
| << st.ToString(); | ||
| } | ||
| } | ||
|
|
||
| // Best-effort: try every transport so one failure can't leave the region | ||
| // registered on the others; mirrors unregisterLocalMemoryBatch (#2869). | ||
| int first_error = 0; | ||
|
|
@@ -1056,6 +1074,18 @@ int TransferEngineImpl::registerLocalMemoryBatch( | |
|
|
||
| int TransferEngineImpl::unregisterLocalMemoryBatch( | ||
| const std::vector<void*>& addr_list) { | ||
| if (multi_transports_) { | ||
| for (auto* addr : addr_list) { | ||
| Status st = multi_transports_->waitMemoryIdle(addr, /*length=*/0, | ||
| /*timeout_ms=*/5000); | ||
| if (st.IsBatchBusy()) { | ||
| LOG(ERROR) << "unregisterLocalMemoryBatch: buffer still in " | ||
| "flight, refusing deregister addr=" | ||
| << addr; | ||
| return ERR_BATCH_BUSY; | ||
| } | ||
| } | ||
| } | ||
| int first_error = 0; | ||
| for (auto transport : multi_transports_->listTransports()) { | ||
| int ret = transport->unregisterLocalMemoryBatch(addr_list); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[P2] Mix pointer bits before selecting a shard
BatchIDis the address returned bynew BatchDesc, so its low bits are constrained by alignment. Taking the raw pointer modulo 64 cannot distribute allocations over all 64 stripes: with 16-byte alignment only buckets 0, 16, 32, and 48 are possible, and stronger alignment can concentrate them further. An ordinary-pointer allocation fixture used only 4/64 buckets on this host; that is an alignment demonstration, not a full-engine benchmark. Please remove the guaranteed alignment bits and mix the remaining address bits before choosing the stripe, and validate the distribution/contention benchmark with actual allocated BatchIDs.