Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions mooncake-transfer-engine/include/multi_transport.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,15 @@
#ifndef MULTI_TRANSPORT_H_
#define MULTI_TRANSPORT_H_

#include <array>
#include <atomic>
#include <condition_variable>
#include <functional>
#include <map>
#include <mutex>
#include <thread>
#include <unordered_map>
#include <unordered_set>
#include <vector>

#include "transport/transport.h"
Expand Down Expand Up @@ -124,6 +126,13 @@ class MultiTransport {

std::vector<Transport *> listTransports();

// Block until no live/deferred batch still references [addr,
// addr+length). Used by unregisterLocalMemory so ibv_dereg_mr
// cannot race outstanding RDMA WRs (local protection → sticky
// Device-not-active). length==0 matches exact source pointer.
// Returns BatchBusy if timeout_ms elapses while still busy.
Status waitMemoryIdle(void *addr, size_t length, int64_t timeout_ms);

void *getBaseAddr();

private:
Expand Down Expand Up @@ -167,6 +176,14 @@ class MultiTransport {

void deferredCleanupLoop();

void trackLiveBatch(BatchID batch_id);
void untrackLiveBatch(BatchID batch_id);
bool memoryReferencedByBatch(BatchID batch_id, void *addr,
size_t length) const;
// Scans live batches. A clean result is confirmed under every shard
// lock so a batch inserted into an already-scanned shard is not missed.
bool memoryBusy(void *addr, size_t length) const;

// Stop a future or already-published event record from being delivered.
// Does not delete the batch.
void suppressCompletion(BatchDesc *batch);
Expand All @@ -189,6 +206,19 @@ class MultiTransport {
std::unordered_map<BatchID, std::shared_ptr<BatchDesc>> batch_desc_set_;

std::mutex deferred_cleanup_mutex_;
// Always-on live batch index for unregister quiesce (#5481).
// Distinct from optional CONFIG_USE_BATCH_DESC_SET ownership map.
// Sharded so concurrent allocate/free only contend on one stripe;
// waitMemoryIdle (unregister) is the only full scan.
static constexpr size_t kLiveBatchShards = 64;
struct LiveBatchShard {
mutable std::mutex mu;
std::unordered_set<BatchID> batches;
};
std::array<LiveBatchShard, kLiveBatchShards> live_batch_shards_{};
static size_t liveBatchShard(BatchID batch_id) {
return static_cast<size_t>(batch_id % kLiveBatchShards);

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Mix pointer bits before selecting a shard

BatchID is the address returned by new BatchDesc, so its low bits are constrained by alignment. Taking the raw pointer modulo 64 cannot distribute allocations over all 64 stripes: with 16-byte alignment only buckets 0, 16, 32, and 48 are possible, and stronger alignment can concentrate them further. An ordinary-pointer allocation fixture used only 4/64 buckets on this host; that is an alignment demonstration, not a full-engine benchmark. Please remove the guaranteed alignment bits and mix the remaining address bits before choosing the stripe, and validate the distribution/contention benchmark with actual allocated BatchIDs.

}
std::condition_variable deferred_cleanup_cv_;
std::unordered_map<BatchID, std::function<void()>>
deferred_cleanup_batches_;
Expand Down
92 changes: 92 additions & 0 deletions mooncake-transfer-engine/src/multi_transport.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ MultiTransport::BatchID MultiTransport::allocateBatchID(size_t batch_size) {
batch_desc_set_[batch_desc->id] = batch_desc;
batch_desc_lock_.unlock();
#endif
trackLiveBatch(batch_desc->id);
return batch_desc->id;
}

Expand Down Expand Up @@ -246,6 +247,7 @@ Status MultiTransport::tryFreeBatchID(
"BatchID cannot be freed until completion callbacks are "
"quiescent");
}
untrackLiveBatch(batch_id);
// The queue holds its own nodes, so the descriptor can be deleted once
// no callback is still inside it. suppressCompletion keeps a published
// id out of a later drain.
Expand Down Expand Up @@ -1473,4 +1475,94 @@ void* MultiTransport::getBaseAddr() {
return 0;
}

void MultiTransport::trackLiveBatch(BatchID batch_id) {
auto& shard = live_batch_shards_[liveBatchShard(batch_id)];
std::lock_guard<std::mutex> guard(shard.mu);
shard.batches.insert(batch_id);
}

void MultiTransport::untrackLiveBatch(BatchID batch_id) {
auto& shard = live_batch_shards_[liveBatchShard(batch_id)];
std::lock_guard<std::mutex> guard(shard.mu);
shard.batches.erase(batch_id);
}

bool MultiTransport::memoryReferencedByBatch(BatchID batch_id, void* addr,
size_t length) const {
if (!batch_id || batch_id == (BatchID)ERR_MEMORY) return false;
auto& batch_desc = *((BatchDesc*)(batch_id));
const uintptr_t a0 = reinterpret_cast<uintptr_t>(addr);
const uintptr_t a1 = length ? (a0 + length) : (a0 + 1);
for (auto& task : batch_desc.task_list) {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Synchronize reference scanning with task and slice publication

The live-batch shard mutex protects set membership and descriptor deletion, but submitTransfer() does not acquire it when it appends to task_list, assigns task.request, or lets the transport populate slice_list. This scan therefore races those non-atomic reads/writes. It does not require submitting to the buffer being unregistered: unregistering buffer A scans a batch concurrently being populated for buffer B. Holding every shard during the confirmation scan cannot synchronize writers that never take those locks. TSan reports this race in an isolated reproduction of the added scanner. Please publish synchronized, batch-owned reference snapshots, or otherwise serialize scanning with the corresponding mutations.

if (__atomic_load_n(&task.is_finished, __ATOMIC_ACQUIRE)) continue;
if (task.request) {
const uintptr_t s =
reinterpret_cast<uintptr_t>(task.request->source);
const size_t n = task.request->length;
Comment on lines +1498 to +1501

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Do not dereference a borrowed request after submission

submitTransfer() assigns task.request = &entries[i]; the batch does not own those elements. The existing Python async path already calls engine_->submitTransfer(batch_id, {entry}), so that temporary request vector is destroyed immediately after submission. Scanning an unfinished task here then reads freed memory. Since unregister scans every live batch, unregistering an unrelated buffer can trigger this too. An ASan reproduction using the extracted scanner reports heap-use-after-free with the payload and slices still alive. Please capture the referenced address ranges in batch-owned storage during submission instead of reading task.request later.

const uintptr_t s1 = s + (n ? n : 1);
if (s < a1 && a0 < s1) return true;
}
for (auto* slice : task.slice_list) {
if (!slice) continue;
const uintptr_t s = reinterpret_cast<uintptr_t>(slice->source_addr);
const size_t n = slice->length;
const uintptr_t s1 = s + (n ? n : 1);
if (s < a1 && a0 < s1) return true;
}
}
return false;
}

bool MultiTransport::memoryBusy(void* addr, size_t length) const {
auto shard_hits = [&](const LiveBatchShard& shard) {
for (BatchID batch_id : shard.batches) {
if (memoryReferencedByBatch(batch_id, addr, length)) return true;
}
return false;
};
for (const auto& shard : live_batch_shards_) {
std::lock_guard<std::mutex> guard(shard.mu);
if (shard_hits(shard)) return true;
}
// A batch can land in a shard we already released. Confirm once while
// holding every shard in index order (same order as any other full scan).
std::unique_lock<std::mutex> locks[kLiveBatchShards];
for (size_t i = 0; i < kLiveBatchShards; ++i) {
locks[i] = std::unique_lock<std::mutex>(live_batch_shards_[i].mu);
}
for (const auto& shard : live_batch_shards_) {
if (shard_hits(shard)) return true;
}
return false;
}

Status MultiTransport::waitMemoryIdle(void* addr, size_t length,
int64_t timeout_ms) {
if (!addr) return Status::InvalidArgument("addr is null");
const auto deadline =
std::chrono::steady_clock::now() +
std::chrono::milliseconds(std::max<int64_t>(timeout_ms, 0));
for (;;) {
bool busy = false;
busy = memoryBusy(addr, length);
if (!busy) {
std::lock_guard<std::mutex> def_guard(deferred_cleanup_mutex_);
for (const auto& entry : deferred_cleanup_batches_) {
if (memoryReferencedByBatch(entry.first, addr, length)) {
busy = true;
break;
}
}
}
if (!busy) return Status::OK();
if (std::chrono::steady_clock::now() >= deadline) {
return Status::BatchBusy(
"unregisterLocalMemory blocked: transfers still reference "
"buffer");
}
deferred_cleanup_cv_.notify_one();
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
}

} // namespace mooncake
30 changes: 30 additions & 0 deletions mooncake-transfer-engine/src/transfer_engine_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -840,6 +840,24 @@ int TransferEngineImpl::registerLocalMemory(void* addr, size_t length,

int TransferEngineImpl::unregisterLocalMemory(void* addr,
bool update_metadata) {
// Quiesce in-flight batches that still reference this buffer before
// transport deregister. Mid-flight ibv_dereg_mr causes LOC_PROT storms
// that latch sticky "Device is not active" (#5481).
if (multi_transports_) {
Status st = multi_transports_->waitMemoryIdle(addr, /*length=*/0,
/*timeout_ms=*/5000);
Comment on lines +847 to +848

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] Check the full registered region before deregistering

Passing length=0 makes memoryReferencedByBatch() check only [addr, addr+1). If a 4096-byte registration at base has an unfinished transfer at base+1024 with length 128, neither the request nor its slices overlap that one-byte interval. This returns OK and proceeds to deregister the MR while it is still in use, leaving the original LOC_PROT failure possible. The batch-unregister path passes zero as well. Please retrieve the registered region's full length and use it for the overlap check in both paths. With the added scanner functions extracted unchanged, this case returns OK for length 0 and BatchBusy for length 4096.

if (st.IsBatchBusy()) {
LOG(ERROR) << "unregisterLocalMemory: buffer still in flight, "
"refusing deregister addr="
<< addr;
return ERR_BATCH_BUSY;
}
if (!st.ok()) {
LOG(WARNING) << "unregisterLocalMemory waitMemoryIdle: "
<< st.ToString();
}
}

// Best-effort: try every transport so one failure can't leave the region
// registered on the others; mirrors unregisterLocalMemoryBatch (#2869).
int first_error = 0;
Expand Down Expand Up @@ -1056,6 +1074,18 @@ int TransferEngineImpl::registerLocalMemoryBatch(

int TransferEngineImpl::unregisterLocalMemoryBatch(
const std::vector<void*>& addr_list) {
if (multi_transports_) {
for (auto* addr : addr_list) {
Status st = multi_transports_->waitMemoryIdle(addr, /*length=*/0,
/*timeout_ms=*/5000);
if (st.IsBatchBusy()) {
LOG(ERROR) << "unregisterLocalMemoryBatch: buffer still in "
"flight, refusing deregister addr="
<< addr;
return ERR_BATCH_BUSY;
}
}
}
int first_error = 0;
for (auto transport : multi_transports_->listTransports()) {
int ret = transport->unregisterLocalMemoryBatch(addr_list);
Expand Down
Loading