diff --git a/src/cuda/op/copy.cc b/src/cuda/op/copy.cc index a7478da855..53653ec7d2 100644 --- a/src/cuda/op/copy.cc +++ b/src/cuda/op/copy.cc @@ -498,10 +498,33 @@ struct Copy { }; struct Im2Col { + static Stmt LowerSIMT(const Im2ColOpNode &op, const LowerArgs &lower_args, + arith::Analyzer *analyzer); static Stmt Lower(const Im2ColOpNode &op, const LowerArgs &lower_args, arith::Analyzer *analyzer); }; +Stmt Im2Col::LowerSIMT(const Im2ColOpNode &op, const LowerArgs &lower_args, + arith::Analyzer *analyzer) { + Stmt lowered = LowerIm2ColSIMT(op, lower_args, analyzer); + auto async_annotation = + op.annotations_.Get(attr::kParallelAsyncWithoutAsyncCommitWait); + bool async_without_async_commit_wait = false; + if (async_annotation) { + if (auto value = async_annotation.value().as()) { + async_without_async_commit_wait = value.value()->value; + } + } + if (!async_without_async_commit_wait || + !TargetCudaHasAsyncCopy(lower_args.target)) { + return lowered; + } + auto result = + InjectPTXAsyncCopy(lowered, /*async_without_async_commit_wait=*/true); + // The WS release arrival also publishes synchronous fallback stores. + return result.stmt; +} + void Copy::CollectFragmentLayouts(const PrimExpr &expr, const Map &bind_var_to_expr, const LayoutMap &existing_layouts, @@ -2827,12 +2850,20 @@ bool RegisterCudaCopy() { const bool cuda_copy_registered = RegisterCudaCopy(); bool RegisterCudaIm2Col() { + RegisterIm2ColImpl(Im2ColImpl{ + "cuda.Im2ColSIMT", + MatchCudaCopyTarget, + 50, + Im2ColImplKind::kSimt, + cuda::Im2Col::LowerSIMT, + }); RegisterIm2ColImpl(Im2ColImpl{ "cuda.Im2Col", [](Target target) { return MatchCudaCopyTarget(target) && TargetIsHopper(target); }, 100, + Im2ColImplKind::kTma, cuda::Im2Col::Lower, }); return true; diff --git a/src/cuda/transform/producer_consumer_ws.cc b/src/cuda/transform/producer_consumer_ws.cc index ef56324f2f..d1ca0f5a5e 100644 --- a/src/cuda/transform/producer_consumer_ws.cc +++ b/src/cuda/transform/producer_consumer_ws.cc @@ -621,6 +621,13 @@ class SimtProducerAnnotator : public StmtExprMutator { PrimExpr VisitExpr_(const CallNode *op) final { static const Op ©_op = Op::Get("tl.tileop.copy"); Call call = Downcast(StmtExprMutator::VisitExpr_(op)); + auto tile_op = ParseOperator(call); + if (tile_op.defined() && tile_op.as() != nullptr && + GetIm2ColImplKind(target_) == Im2ColImplKind::kSimt) { + auto annotations = call->annotations; + annotations.Set(attr::kParallelAsyncWithoutAsyncCommitWait, Bool(true)); + return Call(call->dtype, call->op, call->args, annotations, call->span); + } if (!call->op.same_as(copy_op) || !CanUsePipelineManagedCPAsyncCopy(call)) { return call; } @@ -873,32 +880,32 @@ static Optional ExtractProducerWriteBufferData(const Stmt &stmt) { return Optional(); } -static int -FindFirstAsyncProducerConsumerRead(const Stmt &producer_stmt, - const Array &consumer_compute_stmts, - const BufferDataToBufferMap &buffer_map) { - int earliest_read = static_cast(consumer_compute_stmts.size()); - auto update_earliest_read = [&](const Var &buffer_data) { - for (size_t ci = 0; ci < static_cast(earliest_read); ++ci) { - BufferDataAccessInfo access = AnalyzeBufferDataAccess( - consumer_compute_stmts[ci], buffer_data, buffer_map); - if (access.read) { - earliest_read = static_cast(ci); - return; +static std::pair +FindProducerBufferAccess(const Stmt &producer_stmt, + const Array &access_stmts, + const BufferDataToBufferMap &buffer_map, + std::vector *accesses_producer) { + int earliest_access = static_cast(access_stmts.size()); + int latest_access = -1; + auto update_access = [&](const Var &buffer_data) { + for (size_t ci = 0; ci < access_stmts.size(); ++ci) { + BufferDataAccessInfo access = + AnalyzeBufferDataAccess(access_stmts[ci], buffer_data, buffer_map); + if (access.HasAnyAccess()) { + earliest_access = std::min(earliest_access, static_cast(ci)); + latest_access = std::max(latest_access, static_cast(ci)); + (*accesses_producer)[ci] = true; } } }; if (Optional write_buffer_data = ExtractProducerWriteBufferData(producer_stmt)) { - update_earliest_read(write_buffer_data.value()); + update_access(write_buffer_data.value()); } PostOrderVisit(producer_stmt, [&](const ObjectRef &obj) { - if (earliest_read == 0) { - return; - } if (const auto *store = obj.as()) { if (IsSharedBuffer(store->buffer)) { - update_earliest_read(store->buffer->data); + update_access(store->buffer->data); } return; } @@ -908,12 +915,9 @@ FindFirstAsyncProducerConsumerRead(const Stmt &producer_stmt, return; } PostOrderVisit(call->args[0], [&](const ObjectRef &ptr_obj) { - if (earliest_read == 0) { - return; - } if (const auto *load = ptr_obj.as()) { if (IsSharedBuffer(load->buffer)) { - update_earliest_read(load->buffer->data); + update_access(load->buffer->data); } return; } @@ -927,11 +931,11 @@ FindFirstAsyncProducerConsumerRead(const Stmt &producer_stmt, } auto it = buffer_map.find(ffi::GetRef(var)); if (it != buffer_map.end() && IsSharedBuffer(it->second)) { - update_earliest_read(it->second->data); + update_access(it->second->data); } }); }); - return earliest_read; + return {earliest_access, latest_access}; } static Stmt RewritePreludeTmaProducerStmt(const Stmt &stmt, @@ -1217,6 +1221,20 @@ class ProducerConsumerWSRewriter : public StmtExprMutator { buffer_data_to_buffer_ = BufferDataToBufferCollector::Collect(orig_block->body); + for (size_t i = 0; i < flat_stmts.size(); ++i) { + if (!IsProducer(kinds[i])) { + continue; + } + std::vector target_accesses(flat_stmts.size(), false); + FindProducerBufferAccess(flat_stmts[i], flat_stmts, + buffer_data_to_buffer_, &target_accesses); + for (size_t j = 0; j < flat_stmts.size(); ++j) { + // The split reorders producers ahead of consumers and each other. + if (j != i && target_accesses[j] && (j < i || IsProducer(kinds[j]))) { + return StmtExprMutator::VisitStmt_(orig_realize); + } + } + } Array consumer_compute_stmts; for (size_t i = 0; i < flat_stmts.size(); ++i) { if (!IsProducer(kinds[i])) { @@ -1224,9 +1242,12 @@ class ProducerConsumerWSRewriter : public StmtExprMutator { } } + std::vector consumer_accesses_producer(consumer_compute_stmts.size(), + false); Array prelude_stmts = CollectPreludeStmtsToPipelineLoop(orig_block->body, pipeline_loop); std::vector prelude_tma_plans; + VarSet prelude_write_data; for (const Stmt &stmt : prelude_stmts) { if (ClassifyStmt(stmt, target_) != TileStmtKind::kTmaProducer) { continue; @@ -1235,20 +1256,33 @@ class ProducerConsumerWSRewriter : public StmtExprMutator { if (!write_buffer_data.defined()) { continue; } - int first_read = -1; + int first_access = -1; for (size_t ci = 0; ci < consumer_compute_stmts.size(); ++ci) { BufferDataAccessInfo access = AnalyzeBufferDataAccess( consumer_compute_stmts[ci], write_buffer_data.value(), buffer_data_to_buffer_); - if (access.read) { - first_read = static_cast(ci); - break; + if (access.HasAnyAccess()) { + if (first_access < 0) { + first_access = static_cast(ci); + } + consumer_accesses_producer[ci] = true; } } - if (first_read < 0) { + if (first_access < 0) { continue; } - prelude_tma_plans.push_back({stmt, first_read}); + if (!prelude_write_data.insert(write_buffer_data.value()).second) { + return StmtExprMutator::VisitStmt_(orig_realize); + } + std::vector target_accesses(flat_stmts.size(), false); + FindProducerBufferAccess(stmt, flat_stmts, buffer_data_to_buffer_, + &target_accesses); + for (size_t i = 0; i < flat_stmts.size(); ++i) { + if (IsProducer(kinds[i]) && target_accesses[i]) { + return StmtExprMutator::VisitStmt_(orig_realize); + } + } + prelude_tma_plans.push_back({stmt, first_access}); } int total_barriers = num_fwd + num_bp + prelude_tma_plans.size(); @@ -1267,49 +1301,53 @@ class ProducerConsumerWSRewriter : public StmtExprMutator { Optional write_buffer_data = ExtractProducerWriteBufferData(flat_stmts[i]); if (write_buffer_data.defined()) { - int first_read = -1; + int first_access = -1; int last_access = -1; for (size_t ci = 0; ci < consumer_compute_stmts.size(); ++ci) { BufferDataAccessInfo access = AnalyzeBufferDataAccess( consumer_compute_stmts[ci], write_buffer_data.value(), buffer_data_to_buffer_); - if (access.read && first_read < 0) { - first_read = static_cast(ci); - } if (access.HasAnyAccess()) { + if (first_access < 0) { + first_access = static_cast(ci); + } last_access = static_cast(ci); + consumer_accesses_producer[ci] = true; } } - if (first_read >= 0) { - wait_insert_pos[access_group_idx] = first_read; - arrive_insert_pos[access_group_idx] = last_access + 1; - } else if (last_access >= 0) { - wait_insert_pos[access_group_idx] = 0; + if (first_access >= 0) { + wait_insert_pos[access_group_idx] = first_access; arrive_insert_pos[access_group_idx] = last_access + 1; } } ++access_group_idx; } - // --- Adjust wait positions for SIMT/cp.async producers --- - // SIMT and cp.async producers tie their completion to all forward barriers. - // If a consumer reads any such shared destination before the first TMA - // read, pull all waits earlier so the async producer is also covered. + // SIMT/cp.async producers share every forward barrier and may reuse a + // stage after the first back-pressure wait, so all groups cover their + // first access through their final access. if (has_simt_producer || has_cp_async_producer) { - int earliest_async_read = static_cast(consumer_compute_stmts.size()); + int earliest_async_access = + static_cast(consumer_compute_stmts.size()); + int latest_async_access = -1; for (size_t i = 0; i < flat_stmts.size(); ++i) { if (kinds[i] != TileStmtKind::kSimtProducer && kinds[i] != TileStmtKind::kCpAsyncProducer && kinds[i] != TileStmtKind::kCpAsyncRaw) { continue; } - int first_read = FindFirstAsyncProducerConsumerRead( - flat_stmts[i], consumer_compute_stmts, buffer_data_to_buffer_); - earliest_async_read = std::min(earliest_async_read, first_read); + auto [first_access, last_access] = FindProducerBufferAccess( + flat_stmts[i], consumer_compute_stmts, buffer_data_to_buffer_, + &consumer_accesses_producer); + earliest_async_access = std::min(earliest_async_access, first_access); + latest_async_access = std::max(latest_async_access, last_access); } - // Pull all wait positions earlier if needed. + // Cover the complete shared-stage lifetime in every group. for (int g = 0; g < num_producer_groups; ++g) { - wait_insert_pos[g] = std::min(wait_insert_pos[g], earliest_async_read); + wait_insert_pos[g] = + std::min(wait_insert_pos[g], earliest_async_access); + arrive_insert_pos[g] = + std::max(arrive_insert_pos[g], latest_async_access + 1); } } @@ -1362,6 +1400,10 @@ class ProducerConsumerWSRewriter : public StmtExprMutator { bool all_movable = true; LocalLiveSet movable_probe_live = producer_body_live; for (int ci = compute_cursor; ci < wait_pos; ++ci) { + if (consumer_accesses_producer[ci]) { + all_movable = false; + break; + } const LocalAccessSummary &summary = consumer_compute_summaries[ci]; if (consumer_compute_stmts[ci].as()) { if (!movable_probe_live.NeedsAnyDef(summary)) { @@ -1421,6 +1463,62 @@ class ProducerConsumerWSRewriter : public StmtExprMutator { compute_cursor = wait_pos; } + VarSet prefix_binding_vars; + for (const auto &prefix : producer_loop_prefix_stmts) { + for (const auto &stmt : prefix) { + if (const auto *bind = stmt.as()) { + prefix_binding_vars.insert(bind->var); + } + } + } + // Async producers are issued before the first loop prefix. + LocalLiveSet hoisted_binding_live; + for (size_t i = 0; i < flat_stmts.size(); ++i) { + if (kinds[i] == TileStmtKind::kSimtProducer || + kinds[i] == TileStmtKind::kCpAsyncProducer || + kinds[i] == TileStmtKind::kCpAsyncRaw) { + hoisted_binding_live.AddUses(LocalAccessCollector::Collect( + flat_stmts[i], buffer_data_to_buffer_)); + } + } + LocalLiveSet producer_binding_live = producer_body_live; + VarSet producer_binding_vars; + std::vector producer_bindings; + for (int ci = static_cast(consumer_compute_stmts.size()) - 1; ci >= 0; + --ci) { + const Stmt &stmt = consumer_compute_stmts[ci]; + const auto *bind = stmt.as(); + if (!bind || !producer_binding_live.vars.count(bind->var)) { + continue; + } + producer_binding_live.AddUses(consumer_compute_summaries[ci]); + if (SideEffect(bind->value) > CallEffectKind::kPure) { + // Hoisting a stateful value could cross its original synchronization. + // A loop prefix keeps it in order unless an async producer or a + // hoisted binding needs it first. + if (!prefix_binding_vars.count(bind->var) || + hoisted_binding_live.vars.count(bind->var)) { + return StmtExprMutator::VisitStmt_(orig_realize); + } + continue; + } + producer_binding_vars.insert(bind->var); + producer_bindings.push_back(stmt); + hoisted_binding_live.AddUses(consumer_compute_summaries[ci]); + } + Array producer_scalar_bindings(producer_bindings.rbegin(), + producer_bindings.rend()); + for (auto &prefix : producer_loop_prefix_stmts) { + Array remaining; + for (const auto &stmt : prefix) { + const auto *bind = stmt.as(); + if (!bind || !producer_binding_vars.count(bind->var)) { + remaining.push_back(stmt); + } + } + prefix = std::move(remaining); + } + bool producer_needs_full_thread_extent = false; for (size_t i = 0; i < flat_stmts.size() && !producer_needs_full_thread_extent; ++i) { @@ -1451,8 +1549,9 @@ class ProducerConsumerWSRewriter : public StmtExprMutator { // --- Compute arrive_counts (after producer_extent is finalized) --- bool has_async_producer = has_simt_producer || has_cp_async_producer; - bool has_producer_prefix = - !outer_leading_bindings.empty() || !inner_leading_bindings.empty(); + bool has_producer_prefix = !outer_leading_bindings.empty() || + !inner_leading_bindings.empty() || + !producer_scalar_bindings.empty(); for (const auto &prefix : producer_loop_prefix_stmts) has_producer_prefix |= !prefix.empty(); bool producer_is_single_warp = !has_async_producer && @@ -1509,7 +1608,7 @@ class ProducerConsumerWSRewriter : public StmtExprMutator { if (kinds[i] == TileStmtKind::kSimtProducer) { // Annotate ForNodes with kParallelAsyncWithoutAsyncCommitWait so // InjectPTXAsyncCopy (called from LowerTileOp) does not insert - // commit+wait — the WS pass will emit its own commit+barrier_noinc. + // commit+wait — the WS pass owns commit and completion tracking. simt_producer_stmts.push_back( SimtProducerAnnotator::Annotate(flat_stmts[i], target_)); } else if (kinds[i] == TileStmtKind::kCpAsyncProducer || @@ -1519,7 +1618,7 @@ class ProducerConsumerWSRewriter : public StmtExprMutator { } // Second pass: build the producer body with correct ordering. - Array producer_stmts; + Array producer_stmts = producer_scalar_bindings; int tma_idx = 0; int last_tma_idx = num_producer_groups - 1; bool simt_stmts_emitted = false; @@ -1598,21 +1697,21 @@ class ProducerConsumerWSRewriter : public StmtExprMutator { producer_stmts.push_back(s); } } - // Async-only producers already contribute one completion per thread. - // With a prefix, also publish any synchronous writes by a regular release. + // SIMT lowering can retain synchronous stores alongside async copies. + // Publish those stores with a regular release arrival. if (!producer_is_single_warp) { for (int fwd_base = 0; fwd_base < num_fwd; fwd_base += num_stages) { PrimExpr fwd_id = IntImm(DataType::Int(32), fwd_base) + p_stage_expr; if (!has_async_producer) { // No async copy: each thread publishes its writes and arrives. producer_stmts.push_back(MakeArriveBarrier(barrier_buf, fwd_id)); - } else if (!has_producer_prefix) { + } else if (!has_producer_prefix && !has_simt_producer) { // Async copy only: completion consumes the thread's arrival credit. producer_stmts.push_back(Evaluate( Call(DataType::Handle(), tl::ptx_cp_async_barrier_noinc(), {MakeBarrierRef(barrier_buf, fwd_id)}))); } else { - // Async copy with prefix: reserve completion, then publish writes. + // Reserve async completion and publish synchronous producer writes. producer_stmts.push_back( Evaluate(Call(DataType::Handle(), builtin::ptx_cp_async_barrier(), {MakeBarrierRef(barrier_buf, fwd_id)}))); diff --git a/src/cuda/transform/ws_analysis.cc b/src/cuda/transform/ws_analysis.cc index 72925c4288..458328118d 100644 --- a/src/cuda/transform/ws_analysis.cc +++ b/src/cuda/transform/ws_analysis.cc @@ -271,12 +271,11 @@ TileStmtKind ClassifyStmt(const Stmt &stmt, Target target) { if (auto *copy = tile_op.as()) { return ClassifyCopy(copy, target); } - // Im2Col lowers to tma_load_im2col on Hopper — treat as TMA - // producer so it goes to the producer warp group. if (tile_op.as()) { - if (TargetIsHopper(target)) { + if (GetIm2ColImplKind(target) == Im2ColImplKind::kTma) { return TileStmtKind::kTmaProducer; } + return TileStmtKind::kSimtProducer; } if (auto gemm = GetGemmInfo(tile_op)) { if (IsTmemBuffer(gemm->accumulator)) { diff --git a/src/cuda/transform/ws_analysis.h b/src/cuda/transform/ws_analysis.h index 51588ffaed..eec57013bc 100644 --- a/src/cuda/transform/ws_analysis.h +++ b/src/cuda/transform/ws_analysis.h @@ -34,10 +34,10 @@ enum class TileStmtKind { kTmaProducer, // TMA load (global->shared tile-op copy) kCpAsyncProducer, // tile-op copy selecting cp.async kCpAsyncRaw, // raw ptx_cp_async / commit_group / wait_group statement - kSimtProducer, // Non-tile-op SIMT copy: For loop writing shared from global - kTmaStore, // TMA bulk store (shared->global tile-op copy) - kTcgen05Mma, // tcgen05 GEMM accumulating in tensor memory - kConsumer, // Everything else (compute, wgmma, plain copies) + kSimtProducer, // SIMT producer: non-tile-op copies and SIMT im2col + kTmaStore, // TMA bulk store (shared->global tile-op copy) + kTcgen05Mma, // tcgen05 GEMM accumulating in tensor memory + kConsumer, // Everything else (compute, wgmma, plain copies) }; /// The Evaluate(Call) inside else-less if / attribute / block wrappers, if diff --git a/src/op/copy.cc b/src/op/copy.cc index e679f73567..8631fe70ec 100644 --- a/src/op/copy.cc +++ b/src/op/copy.cc @@ -178,8 +178,8 @@ Stmt LowerIm2ColForTarget(const Im2ColOpNode &op, const LowerArgs &lower_args, bool MatchAnyIm2ColTarget(Target /*target*/) { return true; } -Stmt LowerIm2ColSIMT(const Im2ColOpNode &op, const LowerArgs &lower_args, - arith::Analyzer *analyzer) { +Stmt LowerIm2ColSIMTImpl(const Im2ColOpNode &op, const LowerArgs &lower_args, + arith::Analyzer *analyzer) { const Buffer &src = op.src_; const Buffer &dst = op.dst_; const BufferRegion &dst_region = op.dstRegion_; @@ -269,7 +269,8 @@ bool RegisterDefaultIm2Col() { "default.Im2Col", MatchAnyIm2ColTarget, 0, - LowerIm2ColSIMT, + Im2ColImplKind::kSimt, + LowerIm2ColSIMTImpl, }); return true; } @@ -293,6 +294,15 @@ void RegisterIm2ColImpl(Im2ColImpl impl) { Im2ColImplRegistry().push_back(impl); } +Im2ColImplKind GetIm2ColImplKind(Target target) { + return ResolveIm2ColImpl(target).kind; +} + +Stmt LowerIm2ColSIMT(const Im2ColOpNode &op, const LowerArgs &lower_args, + arith::Analyzer *analyzer) { + return LowerIm2ColSIMTImpl(op, lower_args, analyzer); +} + // Constructs a Copy operator node from call arguments and annotations. // args[0]: source region, args[1]: destination region // annotations: Map containing common SIMT hints and backend-specific metadata. diff --git a/src/op/copy.h b/src/op/copy.h index a9fab87694..d35dbb9d84 100644 --- a/src/op/copy.h +++ b/src/op/copy.h @@ -225,16 +225,22 @@ class Im2ColOpNode : public TileOperatorNode { TileOperator Clone() const; }; +enum class Im2ColImplKind { kSimt, kTma }; + struct Im2ColImpl { const char *name; CopyTargetPredicate match_target; int priority; + Im2ColImplKind kind; Stmt (*lower)(const Im2ColOpNode &op, const LowerArgs &lower_args, arith::Analyzer *analyzer); }; void RegisterIm2ColImpl(Im2ColImpl impl); +Im2ColImplKind GetIm2ColImplKind(Target target); +Stmt LowerIm2ColSIMT(const Im2ColOpNode &op, const LowerArgs &lower_args, + arith::Analyzer *analyzer); class Im2ColOp : public TileOperator { public: diff --git a/testing/python/transform/test_tilelang_transform_producer_consumer_ws.py b/testing/python/transform/test_tilelang_transform_producer_consumer_ws.py index d63abb4e30..a6b21f2aed 100644 --- a/testing/python/transform/test_tilelang_transform_producer_consumer_ws.py +++ b/testing/python/transform/test_tilelang_transform_producer_consumer_ws.py @@ -2,6 +2,8 @@ import re +import pytest + import tilelang import tilelang.language as T import tilelang.testing @@ -912,6 +914,390 @@ def test_tiled_ws_wide_linear_smem_three_stages(): _run_wide_linear_reduce(kernel, 48, 2, 768, "float32") +def make_im2col_kernel(channels=1, binding="none"): + @T.prim_func + def main( + data: T.Tensor((1, 8, 8, channels), T.float16), + weights: T.Tensor((128,), T.float32), + weights_out: T.Tensor((4, 128), T.float32), + data_out: T.Tensor((4, 16, channels), T.float16), + ): + with T.Kernel(1, threads=128): + weight_shared = T.alloc_shared((128,), T.float32) + data_shared = T.alloc_shared((16, channels), T.float16) + for step in T.Pipelined(4, num_stages=2): + T.copy(weights, weight_shared) + if binding == "pure": + step_base = T.int32(step % 2) + nhw_step = T.int32(step_base + 1) + T.im2col(data, data_shared, nhw_step, 0, 3, 1, 1, 1) + elif binding == "stateful": + nhw_step = T.cast(weight_shared[0], T.int32) + T.im2col(data, data_shared, nhw_step, 0, 3, 1, 1, 1) + elif binding == "global": + nhw_step = T.cast(weights[step], T.int32) + T.im2col(data, data_shared, nhw_step, 0, 3, 1, 1, 1) + else: + T.im2col(data, data_shared, step % 2, 0, 3, 1, 1, 1) + for i in T.Parallel(128): + weights_out[step, i] = weight_shared[i] + for i, j in T.Parallel(16, channels): + data_out[step, i, j] = data_shared[i, j] + + return main + + +def make_overwrite_kernel(mode, destination): + @T.prim_func + def main( + data: T.Tensor((1, 8, 8, 32), T.float16), + weights: T.Tensor((128,), T.float32), + weights_out: T.Tensor((4, 128), T.float32), + data_out: T.Tensor((4, 16, 32), T.float16), + ): + with T.Kernel(1, threads=128): + weight_shared = T.alloc_shared((128,), T.float32) + data_shared = T.alloc_shared((16, 32), T.float16) + for step in T.Pipelined(4, num_stages=2): + T.copy(weights, weight_shared) + T.im2col(data, data_shared, step % 2, 0, 3, 1, 1, 1) + if mode == "fill": + if destination == "simt": + T.fill(data_shared, 0) + else: + T.fill(weight_shared, 0) + else: + if destination == "simt": + for i, j in T.Parallel(16, 32): + data_shared[i, j] = T.float16(0) + else: + for i in T.Parallel(128): + weight_shared[i] = T.float32(0) + for i in T.Parallel(128): + weights_out[step, i] = weight_shared[i] + for i, j in T.Parallel(16, 32): + data_out[step, i, j] = data_shared[i, j] + + return main + + +def make_dual_kernel(binding="none", prelude=False, include_simt=True): + @T.prim_func + def main( + data: T.Tensor((1, 8, 8, 32), T.float16), + weights: T.Tensor((128,), T.float32), + other_weights: T.Tensor((3, 128), T.float32), + weights_out: T.Tensor((4, 128), T.float32), + other_out: T.Tensor((4, 128), T.float32), + data_out: T.Tensor((4, 16, 32), T.float16), + ): + with T.Kernel(1, threads=128): + weight_shared = T.alloc_shared((128,), T.float32) + other_shared = T.alloc_shared((128,), T.float32) + data_shared = T.alloc_shared((16, 32), T.float16) + if prelude: + T.copy(weights, weight_shared) + for step in T.Pipelined(4, num_stages=2): + if not prelude: + T.copy(weights, weight_shared) + if include_simt: + T.im2col(data, data_shared, step % 2, 0, 3, 1, 1, 1) + if binding == "pure": + base_offset = T.int32(step % 2) + other_offset = T.int32(base_offset + 1) + T.copy(other_weights[other_offset, 0], other_shared) + elif binding == "stateful": + other_offset = T.cast(weight_shared[0], T.int32) % 2 + T.copy(other_weights[other_offset, 0], other_shared) + else: + T.copy(other_weights[step % 2, 0], other_shared) + for i in T.Parallel(128): + weight_shared[i] = T.float32(0) + for i in T.Parallel(128): + other_out[step, i] = other_shared[i] + for i in T.Parallel(128): + weights_out[step, i] = weight_shared[i] + if include_simt: + for i, j in T.Parallel(16, 32): + data_out[step, i, j] = data_shared[i, j] + + return main + + +def make_shared_producers(order): + @T.prim_func + def main(data: T.Tensor((1, 8, 8, 32), T.float16), seed: T.Tensor((16, 32), T.float16), out: T.Tensor((4, 16, 32), T.float16)): + with T.Kernel(1, threads=128): + target_shared = T.alloc_shared((16, 32), T.float16) + for step in T.Pipelined(4, num_stages=2): + if order == "tma_first": + T.copy(seed, target_shared) + T.im2col(data, target_shared, step % 2, 0, 3, 1, 1, 1) + else: + T.im2col(data, target_shared, step % 2, 0, 3, 1, 1, 1) + T.copy(seed, target_shared) + for i, j in T.Parallel(16, 32): + out[step, i, j] = target_shared[i, j] + + return main + + +def make_preceding_consumer(): + @T.prim_func + def main( + a: T.Tensor((128,), T.float32), + b: T.Tensor((128,), T.float32), + out_a: T.Tensor((4, 128), T.float32), + out_b: T.Tensor((4, 128), T.float32), + ): + with T.Kernel(1, threads=128): + a_shared = T.alloc_shared((128,), T.float32) + target_shared = T.alloc_shared((128,), T.float32) + for step in T.Pipelined(4, num_stages=2): + T.copy(a, a_shared) + for i in T.Parallel(128): + target_shared[i] = T.float32(0) + T.copy(b, target_shared) + for i in T.Parallel(128): + out_a[step, i] = a_shared[i] + out_b[step, i] = target_shared[i] + + return main + + +def touches(expr, buffer_name): + found = [] + tvm.tirx.stmt_functor.post_order_visit( + expr, + lambda node: found.append(node) if isinstance(node, tvm.tirx.BufferLoad) and node.buffer.name.startswith(buffer_name) else None, + ) + return bool(found) + + +def same_barrier(left, right): + if not isinstance(left, tvm.tirx.BufferLoad) or not isinstance(right, tvm.tirx.BufferLoad): + raise RuntimeError("expected a buffer-load barrier reference") + return left.buffer.data.same_as(right.buffer.data) and tvm.ir.structural_equal(left.indices, right.indices, map_free_vars=True) + + +def target_barriers(body, producer, buffer_name, destination): + forward = [] + phases = [] + reuse = [] + last_wait = [] + + def collect_forward(node): + if not isinstance(node, tvm.tirx.Call): + return + name = getattr(node.op, "name", "") + if destination == "simt": + if name.endswith("ptx_cp_async_barrier"): + forward.append(node.args[0]) + if name == "tl.tileop.im2col" and touches(node.args[1], buffer_name): + phases.append(node.annotations["tl.pipeline_mbar_phase_expr"]) + elif name == "tl.tileop.tma_copy" and touches(node.args[1], buffer_name): + if "barrier" not in node.annotations: + raise RuntimeError("target TMA copy has no completion barrier") + forward.append(node.annotations["barrier"]) + phases.append( + node.annotations["tl.pipeline_mbar_phase_expr"] + if "tl.pipeline_mbar_phase_expr" in node.annotations + else tvm.tirx.IntImm("int32", 0) + ) + + tvm.tirx.stmt_functor.post_order_visit(producer if destination == "simt" else body, collect_forward) + + def collect_reuse(node): + if not isinstance(node, tvm.tirx.Call): + return + name = getattr(node.op, "name", "") + if name.endswith("mbarrier_wait_parity"): + last_wait[:] = [node.args[0]] + matches = (destination == "tma" and name == "tl.tileop.tma_copy") or (destination == "simt" and name == "tl.tileop.im2col") + if matches and touches(node.args[1], buffer_name): + if not last_wait: + raise RuntimeError("target producer has no preceding reuse wait") + reuse.append(last_wait[0]) + + tvm.tirx.stmt_functor.post_order_visit(producer, collect_reuse) + if len(forward) != 1 or len(phases) != 1 or len(reuse) > 1: + raise RuntimeError(f"expected one target completion barrier, got {len(forward)} forward/{len(reuse)} reuse") + return forward[0], phases[0], reuse[0] if reuse else None + + +def events(stmt, buffer_name, forward, phase, reuse): + result = [] + + def visit(node): + if isinstance(node, tvm.tirx.BufferLoad) and node.buffer.name.startswith(buffer_name): + result.append("read") + if isinstance(node, tvm.tirx.BufferStore) and node.buffer.name.startswith(buffer_name): + result.append("overwrite") + if isinstance(node, tvm.tirx.Call): + name = getattr(node.op, "name", "") + if name.endswith("mbarrier_wait_parity"): + matches = same_barrier(node.args[0], forward) and tvm.ir.structural_equal(node.args[1], phase, map_free_vars=True) + result.append("target_wait" if matches else "other_wait") + if name.endswith("ptx_arrive_barrier") and reuse is not None and same_barrier(node.args[0], reuse): + result.append("target_release") + if name == "tl.tileop.fill" and touches(node.args[0], buffer_name): + result.append("overwrite") + + tvm.tirx.stmt_functor.post_order_visit(stmt, visit) + return result + + +def ordering_violations(producer, consumer, reuse): + violations = [] + if "overwrite" in producer: + violations.append("consumer overwrite moved into producer issue sequence") + access_positions = [i for i, event in enumerate(consumer) if event in ("read", "overwrite")] + if access_positions and ("target_wait" not in consumer or access_positions[0] < consumer.index("target_wait")): + violations.append("consumer access precedes its target completion wait") + if ( + reuse is not None + and access_positions + and ("target_release" not in consumer or consumer.index("target_release") < access_positions[-1]) + ): + violations.append("target stage released before its final consumer access") + return violations + + +def is_specialized(func): + return "tl_tiled_ws_applied" in func.attrs + + +def producer_consumer_split(func): + def has_tma(stmt): + calls = [] + tvm.tirx.stmt_functor.post_order_visit( + stmt, + lambda node: ( + calls.append(node) if isinstance(node, tvm.tirx.Call) and getattr(node.op, "name", "") == "tl.tileop.tma_copy" else None + ), + ) + return bool(calls) + + splits = [] + + def visit(node): + if isinstance(node, tvm.tirx.IfThenElse) and node.else_case is not None and has_tma(node.then_case): + splits.append(node) + + tvm.tirx.stmt_functor.post_order_visit(func.body, visit) + if len(splits) != 1: + raise RuntimeError(f"expected one producer/consumer split, got {len(splits)}") + return splits[0] + + +def check_fallback(before, after): + if is_specialized(after): + raise RuntimeError("dependent producer work was warp-specialized") + + def remove_pipeline_hint(node): + if isinstance(node, tvm.tirx.For) and "num_stages" in node.annotations: + annotations = {key: value for key, value in node.annotations.items() if key != "num_stages"} + return tvm.tirx.For( + node.loop_var, + node.min, + node.extent, + node.kind, + node.body, + thread_binding=node.thread_binding, + annotations=annotations, + step=node.step, + ) + return None + + expected_body = tvm.tirx.stmt_functor.ir_transform(before.body, None, remove_pipeline_hint, ["tirx.For"]) + if not tvm.ir.structural_equal(before.with_body(expected_body), after): + raise RuntimeError("fallback changed the original loop beyond its num_stages hint") + + +def lower_producer_consumer_scenario(func): + target = determine_target({"kind": "cuda", "arch": "sm_100"}, return_object=True) + mod = tvm.IRModule.from_expr(func.with_attr("global_symbol", "main")) + mod = tvm.tirx.transform.BindTarget(target)(mod) + before = tilelang.transform.MaterializeKernelLaunch()(mod) + after = tilelang.cuda.transform.ProducerConsumerWarpSpecialized()(before) + scoped = tvm.tirx.transform.ConvertSSA()(after)["main"] + bound = [*scoped.params, *(buf.data for buf in scoped.buffer_map.values())] + undefined = tvm.tirx.analysis.undefined_vars(scoped.body, bound) + if undefined: + pytest.fail(f"transformed kernel references undefined variables: {undefined}") + with tvm.transform.PassContext(), target: + artifact = tilelang.lower(func, target=target) + if not artifact.kernel_source: + pytest.fail("compiler returned no CUDA source") + return before["main"], after["main"] + + +def check_consumer_accesses(func, buffer_name, destination, overwrite=False): + if not is_specialized(func): + pytest.fail("independent producers lost warp specialization") + split = producer_consumer_split(func) + forward, phase, reuse = target_barriers(func.body, split.then_case, buffer_name, destination) + producer = events(split.then_case, buffer_name, forward, phase, reuse) + consumer = events(split.else_case, buffer_name, forward, phase, reuse) + violations = ordering_violations(producer, consumer, reuse) + if violations: + pytest.fail(str(violations)) + if "read" not in consumer: + pytest.fail("consumer read was lost from the transformed IR") + if overwrite and "overwrite" not in consumer: + pytest.fail("consumer overwrite was lost from the transformed IR") + + +@tilelang.testing.requires_cuda(support_required="compile-only") +@pytest.mark.parametrize("channels,binding", [(1, "none"), (32, "none"), (32, "pure"), (32, "stateful"), (32, "global")]) +def test_tiled_ws_simt_im2col(channels, binding): + before, after = lower_producer_consumer_scenario(make_im2col_kernel(channels, binding)) + if binding in ("stateful", "global"): + check_fallback(before, after) + return + check_consumer_accesses(after, "data_shared", "simt") + arrivals = [] + tvm.tirx.stmt_functor.post_order_visit( + producer_consumer_split(after).then_case, + lambda node: ( + arrivals.append(node) + if isinstance(node, tvm.tirx.Call) and getattr(node.op, "name", "").endswith("ptx_arrive_barrier") + else None + ), + ) + if not arrivals: + pytest.fail("SIMT producer has no release arrival for synchronous stores") + + +@tilelang.testing.requires_cuda(support_required="compile-only") +@pytest.mark.parametrize("mode", ["fill", "parallel"]) +@pytest.mark.parametrize("destination", ["simt", "tma"]) +def test_tiled_ws_waits_before_consumer_overwrite(mode, destination): + _, after = lower_producer_consumer_scenario(make_overwrite_kernel(mode, destination)) + check_consumer_accesses(after, "data_shared" if destination == "simt" else "weight_shared", destination, overwrite=True) + + +@tilelang.testing.requires_cuda(support_required="compile-only") +@pytest.mark.parametrize( + "binding,prelude,include_simt", + [("none", False, True), ("pure", False, True), ("pure", False, False), ("none", True, True), ("stateful", False, False)], +) +def test_tiled_ws_multiple_producer_dependencies(binding, prelude, include_simt): + before, after = lower_producer_consumer_scenario(make_dual_kernel(binding, prelude, include_simt)) + if binding == "stateful": + check_fallback(before, after) + else: + check_consumer_accesses(after, "weight_shared", "tma", overwrite=True) + + +@tilelang.testing.requires_cuda(support_required="compile-only") +@pytest.mark.parametrize("order", ["tma_first", "im2col_first", "consumer_before_tma"]) +def test_tiled_ws_preserves_conflicting_producer_order(order): + func = make_preceding_consumer() if order == "consumer_before_tma" else make_shared_producers(order) + before, after = lower_producer_consumer_scenario(func) + check_fallback(before, after) + + if __name__ == "__main__": test_tiled_ws_skips_device_bound_tma_descriptor_base() test_tiled_ws_places_producer_in_first_warp_group()