Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 54 additions & 1 deletion src/cuda/transform/producer_consumer_ws.cc
Original file line number Diff line number Diff line change
Expand Up @@ -192,9 +192,11 @@ struct LocalLiveSet {
return false;
}

void AddUses(const LocalAccessSummary &summary) {
bool AddUses(const LocalAccessSummary &summary) {
size_t num_values = buffers.size() + vars.size();
buffers.insert(summary.read_buffers.begin(), summary.read_buffers.end());
vars.insert(summary.read_vars.begin(), summary.read_vars.end());
return buffers.size() + vars.size() != num_values;
}
};

Expand Down Expand Up @@ -1791,6 +1793,8 @@ class ProducerConsumerWSRewriter : public StmtExprMutator {
dummy_producer, dummy_consumer);
dummy_ws =
AttrStmt(ws_partition, attr::kWarpSpecializationScope, 0, dummy_ws);
guarded_postlude_stmts_ = {};
kept_shared_prelude_stmts_ = {};
Optional<Stmt> replaced =
ReplacePipelineLoopInStmt(orig_block->body, pipeline_loop, dummy_ws,
producer_extent, consumer_extent);
Expand All @@ -1805,6 +1809,11 @@ class ProducerConsumerWSRewriter : public StmtExprMutator {
// LayoutInference can infer an independent producer-side thread range.
BufferRemap producer_buffer_remap;
Array<Buffer> producer_private_buffers;
// A statement after the pipeline is executed by the consumer only (and is
// later sunk into the consumer branch). If it writes a value that the
// producer reads, warp specialization would leave the producer with the
// value of the previous run, so decline the transformation.
bool declined_loop_carried_postlude = false;
{
BufferSet block_alloc_buffers;
for (const auto &buffer : orig_block->alloc_buffers) {
Expand All @@ -1822,6 +1831,30 @@ class ProducerConsumerWSRewriter : public StmtExprMutator {
LocalAccessCollector::Collect(
guard->then_case, buffer_data_to_buffer_));
}
LocalLiveSet producer_reads;
producer_reads.AddUses(producer_access);
// The producer may read a value that a statement kept in the shared
// prelude derives from a branch-private local. Follow those reads back
// through the shared prelude so a post-pipeline update of that local is
// not missed.
bool propagated = true;
while (propagated) {
propagated = false;
for (const auto &stmt : kept_shared_prelude_stmts_) {
LocalAccessSummary summary =
LocalAccessCollector::Collect(stmt, buffer_data_to_buffer_);
if (producer_reads.NeedsAnyDef(summary)) {
propagated |= producer_reads.AddUses(summary);
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
}
for (const auto &postlude : guarded_postlude_stmts_) {
if (producer_reads.NeedsAnyDef(LocalAccessCollector::Collect(
postlude, buffer_data_to_buffer_))) {
declined_loop_carried_postlude = true;
break;
}
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
auto maybe_clone = [&](const Buffer &buffer) {
if (!buffer.defined() ||
!(IsFragmentBuffer(buffer) || IsLocalBuffer(buffer)) ||
Expand All @@ -1841,6 +1874,16 @@ class ProducerConsumerWSRewriter : public StmtExprMutator {
maybe_clone(buffer);
}
}
if (declined_loop_carried_postlude) {
// The producer would read a stale value on the next run of the pipeline.
// Decline; the pass-level fallback drops the pipeline annotations, so
// this block runs as a plain serial loop.
DLOG(WARNING) << "[WS] declined: a post-pipeline statement writes a "
"value the producer reads on the next run";
guarded_postlude_stmts_ = {};
kept_shared_prelude_stmts_ = {};
return GetRef<Stmt>(orig_realize);
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if (!producer_buffer_remap.empty()) {
rewritten_producer =
RemapBuffers(rewritten_producer, producer_buffer_remap);
Expand Down Expand Up @@ -2268,6 +2311,7 @@ class ProducerConsumerWSRewriter : public StmtExprMutator {
new_seq.push_back(rewritten_loop.value());

for (int i = loop_idx + 1; i < static_cast<int>(op->seq.size()); ++i) {
rewriter_->guarded_postlude_stmts_.push_back(op->seq[i]);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
new_seq.push_back(GuardConsumerOnly(op->seq[i]));
}
return new_seq.size() == 1 ? new_seq[0] : SeqStmt(new_seq);
Expand Down Expand Up @@ -2424,6 +2468,7 @@ class ProducerConsumerWSRewriter : public StmtExprMutator {
bool to_consumer = placement == PreludeStmtPlacement::kConsumerOnly ||
placement == PreludeStmtPlacement::kDuplicateToBoth;
if (placement == PreludeStmtPlacement::kKeepSharedPrelude) {
rewriter_->kept_shared_prelude_stmts_.push_back(stmt);
Stmt kept = stmt;
if (auto it = rewriter_->common_prelude_rewrites_.find(stmt);
it != rewriter_->common_prelude_rewrites_.end()) {
Expand Down Expand Up @@ -2527,6 +2572,14 @@ class ProducerConsumerWSRewriter : public StmtExprMutator {
Array<Stmt> extracted_producer_init_;
Array<Stmt> extracted_consumer_init_;
Array<IfThenElse> guarded_producer_prelude_;
// Statements after the pipeline loop that GuardConsumerOnly holds to the
// consumer partition. Checked against the producer's reads so a loop-carried
// scalar update cannot be sunk into the consumer alone.
Array<Stmt> guarded_postlude_stmts_;
// Statements kept in the shared prelude. Their reads are followed when
// checking post-pipeline writes so an indirect producer dependency through a
// shared definition is not missed.
Array<Stmt> kept_shared_prelude_stmts_;
// Buffers touched by statements that stay in the shared prelude. They cannot
// be cloned for the producer: the shared statement runs before the partition
// split, so both branches must keep pointing at the same buffer.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1021,5 +1021,148 @@ def test_ws_producer_prelude_before_outer_loop():
torch.testing.assert_close(kernel(A, B), ref.repeat(2, 1), rtol=1e-2, atol=1e-2)


def cross_run_producer_scalar_update(
M=256,
N=256,
K=256,
block_M=64,
block_N=64,
block_K=32,
num_stages=2,
runs=2,
dtype="float16",
threads=128,
):
"""Issue #3842: the running K offset is updated after each pipeline run.

The producer reads ``start`` when it evaluates the copy indices of the next
run, so the update cannot be sunk into the consumer branch.
"""
iterations = K // block_K
step = (iterations + runs - 1) // runs

@T.prim_func
def main(
A: T.Tensor((M, K), dtype),
B: T.Tensor((K, N), dtype),
C: T.Tensor((M, N), dtype),
):
with T.Kernel(N // block_N, M // block_M, threads=threads) as (bx, by):
A_s = T.alloc_shared((block_M, block_K), dtype)
B_s = T.alloc_shared((block_K, block_N), dtype)
C_l = T.alloc_fragment((block_M, block_N), "float32")
start = T.alloc_var(T.int32, init=0)
T.clear(C_l)
for _ in T.serial(runs):
k0 = T.alloc_var(T.int32, init=start)
k1 = T.alloc_var(T.int32, init=T.min(iterations, k0 + step))
for ko in T.Pipelined(k1 - k0, num_stages=num_stages):
T.copy(A[by * block_M, (k0 + ko) * block_K], A_s)
T.copy(B[(k0 + ko) * block_K, bx * block_N], B_s)
T.gemm(A_s, B_s, C_l)
start = start + (k1 - k0)
T.copy(C_l, C[by * block_M, bx * block_N])

return main


@tilelang.testing.requires_cuda
def test_ws_pass_declines_loop_carried_producer_scalar_update():
"""A post-pipeline write read by the producer's next run must decline WS."""
script = _apply_ws_pass(cross_run_producer_scalar_update())["main"].script()
assert "tl_tiled_ws_applied" not in script, "WS must decline when a post-pipeline update feeds the producer's next run"


@tilelang.testing.requires_cuda
@tilelang.testing.requires_cuda_compute_version(9, 0)
def test_ws_declined_loop_carried_update_matches_reference():
import torch

kernel = tilelang.compile(
cross_run_producer_scalar_update(),
target=determine_target(),
out_idx=[2],
pass_configs={tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: False},
)
torch.manual_seed(0)
A = torch.randn(256, 256, dtype=torch.float16, device="cuda")
B = torch.randn(256, 256, dtype=torch.float16, device="cuda")
ref = A.float() @ B.float()
torch.testing.assert_close(kernel(A, B).float(), ref, rtol=1e-2, atol=1e-2)


def cross_run_producer_shared_prelude_index(
M=256,
N=256,
K=256,
block_M=64,
block_N=64,
block_K=32,
num_stages=2,
runs=2,
dtype="float16",
threads=128,
):
"""Issue #3842, indirect variant: the producer index comes from the prelude.

Unlike ``cross_run_producer_scalar_update``, the K offsets are plain Bind
temporaries derived from ``start``, so they stay in the shared prelude
instead of being copied into the producer branch. The producer still reads
them for its copy indices, which makes ``start`` a producer input through
the prelude even though the producer never reads ``start`` directly.
"""
iterations = K // block_K
step = (iterations + runs - 1) // runs

@T.prim_func
def main(
A: T.Tensor((M, K), dtype),
B: T.Tensor((K, N), dtype),
C: T.Tensor((M, N), dtype),
):
with T.Kernel(N // block_N, M // block_M, threads=threads) as (bx, by):
A_s = T.alloc_shared((block_M, block_K), dtype)
B_s = T.alloc_shared((block_K, block_N), dtype)
C_l = T.alloc_fragment((block_M, block_N), "float32")
start = T.alloc_var(T.int32, init=0)
T.clear(C_l)
for _ in T.serial(runs):
k_off = start
k1 = T.min(iterations, k_off + step)
for ko in T.Pipelined(k1 - k_off, num_stages=num_stages):
T.copy(A[by * block_M, (k_off + ko) * block_K], A_s)
T.copy(B[(k_off + ko) * block_K, bx * block_N], B_s)
T.gemm(A_s, B_s, C_l)
start = start + (k1 - k_off)
T.copy(C_l, C[by * block_M, bx * block_N])

return main


@tilelang.testing.requires_cuda
def test_ws_pass_declines_shared_prelude_producer_index_update():
"""A post-pipeline update read through the shared prelude must decline WS."""
script = _apply_ws_pass(cross_run_producer_shared_prelude_index())["main"].script()
assert "tl_tiled_ws_applied" not in script, "WS must follow producer reads through the shared prelude"


@tilelang.testing.requires_cuda
@tilelang.testing.requires_cuda_compute_version(9, 0)
def test_ws_declined_shared_prelude_update_matches_reference():
import torch

kernel = tilelang.compile(
cross_run_producer_shared_prelude_index(),
target=determine_target(),
out_idx=[2],
pass_configs={tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: False},
)
torch.manual_seed(0)
A = torch.randn(256, 256, dtype=torch.float16, device="cuda")
B = torch.randn(256, 256, dtype=torch.float16, device="cuda")
ref = A.float() @ B.float()
torch.testing.assert_close(kernel(A, B).float(), ref, rtol=1e-2, atol=1e-2)


if __name__ == "__main__":
tilelang.testing.main()