diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b504365a..ff4141636 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,11 +85,16 @@ jobs: tests/contracts/test_tolerance_contract.py \ tests/runtime/test_kernel_registry.py - - name: Run Nemotron Router Contract Tests (CPU-safe) + - name: Run Nemotron Contract Tests (CPU-safe) + # Only CPU-executable suites here: the provider suites under + # tests/backends/{cuda,rocm} are hardware-marked and skip wholesale on + # ubuntu-latest, so listing them would report a green run that executed + # nothing. They run on the GPU CI jobs and in the operator PR's own + # verification record. run: | PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -q \ tests/reference/test_nemotron_router_reference.py \ - tests/backends/cuda/test_nemotron_router_cuda.py + tests/reference/test_nemotron_experts_reference.py - name: Run Attention Ground-Truth Tests (CPU-safe) run: | diff --git a/benchmarks/operators/moe/benchmark_nemotron_experts_training.py b/benchmarks/operators/moe/benchmark_nemotron_experts_training.py new file mode 100644 index 000000000..8003b44c8 --- /dev/null +++ b/benchmarks/operators/moe/benchmark_nemotron_experts_training.py @@ -0,0 +1,540 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Nemotron grouped experts + combine: training-path timing and FP64 qualification. + +Baseline: the same math expert by expert with `torch.matmul` in FP32 and a +weighted, slot-ascending combine - the obvious eager formulation. Its expert span +list is read once outside the timed region. `--baselines grouped_mm` adds a +production-shaped row: `torch._grouped_mm`, the cuBLAS/CUTLASS grouped GEMM that a +serving stack would call, measured forward only because a grouped MoE's weight +gradients need a grouped wgrad kernel that `_grouped_mm` does not expose. + +Candidates: the provider objects themselves (`NemotronExpertsOp`, +`TritonNemotronExpertsOp`), so the public validation wrapper is excluded from +both sides of the comparison. Timing uses CUDA Graphs by default so host launch +gaps do not dominate; `--mode eager` measures the eager path including them. +Cotangents are prepared outside timing. + +The FP64 qualification rebuilds the pipeline in FP64 and reports the relative L2 +error of the forward and every gradient. It is memory bound, so +`--qualify-max-tokens` bounds which rows carry a verdict, and a set whose +estimated FP64 footprint (dominated by the token-independent FP64 weights and +their gradients) exceeds the free device memory is skipped with that reason +instead of OOMing; skipped rows record why they carry none. + +Transformer Engine is not used: the pinned `transformer_engine==2.19.0` grouped +GEMM is not available in this environment, and the documented baseline is the +one that actually ran. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import statistics +from pathlib import Path + +import torch + +import rl_engine.ops.moe.nemotron_experts as core +from rl_engine.reference.moe.nemotron_experts import routed_experts, shared_experts +from rl_engine.reference.moe.nemotron_router import dispatch_tokens, route_scores + +HIDDEN = 2688 +INTERMEDIATE = 1856 +EXPERTS = 128 +SHARED = 3712 +SLOTS = 6 +GRADIENT_NAMES = ("packed", "x", "weights", "w_up", "w_down", "w_sh1", "w_sh2") +DTYPES = {"bf16": torch.bfloat16} + + +def stats_error(actual, expected): + difference = (actual.double() - expected.double()).abs() + return { + "relative_l2": (difference.norm() / expected.double().norm().clamp_min(1e-30)).item(), + "max_abs": difference.max().item(), + } + + +def build(tokens, distribution, seed=434, dtype=torch.bfloat16): + """Router output, payload, stacked weights and a fixed cotangent.""" + generator = torch.Generator().manual_seed(seed) + payload = torch.randn(tokens, HIDDEN, generator=generator) * 0.5 + scores = torch.rand(tokens, EXPERTS, generator=generator) + bias = torch.zeros(EXPERTS) + if distribution == "concentrated": + bias[:SLOTS] = 4.0 + ids, route_weights = route_scores(scores, bias, SLOTS) + permutation, offsets, packed = dispatch_tokens(payload, ids, EXPERTS) + device = "cuda" + case = { + "packed": packed.to(device=device, dtype=dtype), + "x": payload.to(device=device, dtype=dtype), + "weights": route_weights.to(device=device), + "permutation": permutation.to(device=device), + "offsets": offsets.to(device=device), + "w_up": (torch.randn(EXPERTS, INTERMEDIATE, HIDDEN, generator=generator) * 0.05).to( + device=device, dtype=dtype + ), + "w_down": (torch.randn(EXPERTS, HIDDEN, INTERMEDIATE, generator=generator) * 0.02).to( + device=device, dtype=dtype + ), + "w_sh1": (torch.randn(SHARED, HIDDEN, generator=generator) * 0.05).to( + device=device, dtype=dtype + ), + "w_sh2": (torch.randn(HIDDEN, SHARED, generator=generator) * 0.02).to( + device=device, dtype=dtype + ), + "ids": ids, + } + counts = torch.bincount(ids.reshape(-1), minlength=EXPERTS) + active = int((counts > 0).sum()) + steps = torch.Generator().manual_seed(seed + 1) + meta = { + "spans": offsets.tolist(), + # Only experts with rows are ever read: two BF16 weight matrices each. + "weights_bytes": active * (2 * INTERMEDIATE * HIDDEN) * 2, + "n_e": { + "min": int(counts.min()), + "max": int(counts.max()), + "empty_experts": int((counts == 0).sum()), + "active_experts": active, + }, + "cotangent": (torch.randn(tokens, HIDDEN, generator=steps) * 0.01).to( + device=device, dtype=dtype + ), + } + return case, meta + + +def leaves_of(case): + return {name: case[name].detach().clone().requires_grad_(True) for name in GRADIENT_NAMES} + + +def baseline_calls(case, spans, leaves, cotangent): + """Per-expert `torch.matmul` in FP32 plus a slot-ascending weighted combine.""" + packed = case["packed"] + weights = case["weights"] + tokens = case["x"].shape[0] + rows, width = packed.shape + packed_rows = case["permutation"].argsort().reshape(tokens, SLOTS) + + def body(source, source_x, source_weights, up, down, sh1, sh2): + wide = torch.empty((rows, width), device=packed.device, dtype=torch.float32) + for expert in range(EXPERTS): + lo, hi = spans[expert], spans[expert + 1] + if lo == hi: + continue + gate = torch.relu(source[lo:hi].float() @ up[expert].float().t()).square() + wide[lo:hi] = gate @ down[expert].float().t() + routed = torch.zeros((tokens, width), device=packed.device, dtype=torch.float32) + for slot in range(SLOTS): + row = wide[packed_rows[:, slot]] + routed = routed + row * source_weights[:, slot : slot + 1] + shared = torch.relu(source_x.float() @ sh1.float().t()).square() @ sh2.float().t() + return (routed + shared).to(packed.dtype) + + def forward(): + return body( + packed, case["x"], weights, case["w_up"], case["w_down"], case["w_sh1"], case["w_sh2"] + ) + + def backward(): + return torch.autograd.grad( + body( + leaves["packed"], + leaves["x"], + leaves["weights"], + leaves["w_up"], + leaves["w_down"], + leaves["w_sh1"], + leaves["w_sh2"], + ), + [leaves[name] for name in GRADIENT_NAMES], + cotangent, + ) + + return forward, backward + + +def grouped_mm_calls(case): + """Production-shaped baseline: `torch._grouped_mm` (cuBLAS/CUTLASS grouped GEMM). + + Forward only. A grouped MoE's weight gradients need a grouped wgrad kernel + (Transformer Engine's `GroupedLinear` does dgrad + wgrad in one grouped call); + `torch._grouped_mm` groups over the token rows alone, so `dW` - whose reduction + length differs per expert - is not expressible with it. The stacked weights + are materialized in the layout each projection needs outside the timed region, + exactly as a production layer would store them, and the keep-in-FP32 combine + matches the eager baseline so only the GEMM differs. + """ + if not hasattr(torch, "_grouped_mm"): + raise RuntimeError("torch._grouped_mm is not available in this build") + grouped_mm = torch._grouped_mm + packed = case["packed"] + tokens = case["x"].shape[0] + width = packed.shape[1] + device = packed.device + weights = case["weights"] + offs = case["offsets"][1:].to(torch.int32).contiguous() + single = torch.tensor([tokens], dtype=torch.int32, device=device) + # (K, N) per group: the up projection contracts over H, the down over F. + up_t = case["w_up"].transpose(1, 2).contiguous() + down_t = case["w_down"].transpose(1, 2).contiguous() + sh1_t = case["w_sh1"].t().contiguous().unsqueeze(0) + sh2_t = case["w_sh2"].t().contiguous().unsqueeze(0) + packed_rows = case["permutation"].argsort().reshape(tokens, SLOTS) + + def combine(wide, shared): + routed = torch.zeros((tokens, width), device=device, dtype=torch.float32) + for slot in range(SLOTS): + routed = routed + wide[packed_rows[:, slot]].float() * weights[:, slot : slot + 1] + return (routed + shared.float()).to(packed.dtype) + + def forward(): + up = grouped_mm(packed, up_t, offs=offs) + gate = torch.relu(up.float()).square().to(packed.dtype) + wide = grouped_mm(gate, down_t, offs=offs) + shared_up = grouped_mm(case["x"], sh1_t, offs=single) + shared_gate = torch.relu(shared_up.float()).square().to(packed.dtype) + shared = grouped_mm(shared_gate, sh2_t, offs=single) + return combine(wide, shared) + + return forward, None + + +def provider_calls(op, case, leaves, cotangent): + def forward(): + return op( + case["packed"], + case["x"], + case["weights"], + case["permutation"], + case["offsets"], + case["w_up"], + case["w_down"], + case["w_sh1"], + case["w_sh2"], + ) + + def backward(): + return torch.autograd.grad( + op( + leaves["packed"], + leaves["x"], + leaves["weights"], + case["permutation"], + case["offsets"], + leaves["w_up"], + leaves["w_down"], + leaves["w_sh1"], + leaves["w_sh2"], + ), + [leaves[name] for name in GRADIENT_NAMES], + cotangent, + ) + + return forward, backward + + +def qualification_bytes(case): + """Estimate the FP64 qualification's peak device memory. + + Parameters and their gradients dominate and do not depend on the token + count; the routed activations add one term proportional to `rows`. The + estimate is a lower bound with a small margin, used only to skip a + qualification that cannot fit rather than to size an allocation. + """ + parameters = sum( + case[name].numel() + for name in ("w_up", "w_down", "w_sh1", "w_sh2") + if case.get(name) is not None + ) + rows = case["packed"].shape[0] + width = case["packed"].shape[1] + intermediate = case["w_up"].shape[1] + shared = case["w_sh1"].shape[0] if case.get("w_sh1") is not None else 0 + tokens = case["x"].shape[0] + # FP64 parameters + their FP64 gradients, then the routed and token + # activation/backward buffer sets at FP64. + elements = 2 * parameters + 6 * rows * (width + intermediate) + 3 * tokens * (width + shared) + return int(elements * 8 * 1.1) + + +def qualify(op, case, meta, token_limit): + """FP64 comparison of the forward and every gradient on this provider.""" + tokens = case["x"].shape[0] + if tokens > token_limit: + return {"skipped": f"above --qualify-max-tokens={token_limit}"} + needed = qualification_bytes(case) + free_bytes, _ = torch.cuda.mem_get_info() + if needed > free_bytes: + return { + "skipped": f"needs ~{needed / 2**30:.1f} GiB FP64, {free_bytes / 2**30:.1f} GiB free" + } + leaves = leaves_of(case) + cotangent = meta["cotangent"] + args = (case["permutation"], case["offsets"]) + actual = op( + leaves["packed"], + leaves["x"], + leaves["weights"], + *args, + leaves["w_up"], + leaves["w_down"], + leaves["w_sh1"], + leaves["w_sh2"], + ) + actual_grads = torch.autograd.grad(actual, [leaves[name] for name in GRADIENT_NAMES], cotangent) + exact = {name: case[name].detach().double().requires_grad_() for name in GRADIENT_NAMES} + wide = routed_experts(exact["packed"], exact["weights"], *args, exact["w_up"], exact["w_down"]) + wide = wide + shared_experts(exact["x"], exact["w_sh1"], exact["w_sh2"]) + exact_grads = torch.autograd.grad( + wide, [exact[name] for name in GRADIENT_NAMES], cotangent.double() + ) + report = {"forward": stats_error(actual, wide)} + for position, name in enumerate(GRADIENT_NAMES): + report[name] = stats_error(actual_grads[position], exact_grads[position]) + return report + + +def capture(fn): + for _ in range(5): + fn() + torch.cuda.synchronize() + # Measure the increment around capture, not around an eager call: a graph + # keeps its own private pool, so an eager call taken before capture misses + # exactly the memory graph mode adds. Providers are captured in sequence and + # `before` already holds the previous graphs, so the subtraction isolates + # this provider's increment and the numbers stay comparable across providers. + before = torch.cuda.memory_allocated() + torch.cuda.reset_peak_memory_stats() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outputs = fn() + for _ in range(5): + graph.replay() + torch.cuda.synchronize() + peak = torch.cuda.max_memory_allocated() - before + return graph, outputs, peak + + +def paired_measure(calls, reverse, mode): + measured = {} + if mode == "graph": + measured = {name: capture(fn) for name, fn in calls.items()} + else: + for name, fn in calls.items(): + for _ in range(5): + fn() + torch.cuda.synchronize() + before = torch.cuda.memory_allocated() + torch.cuda.reset_peak_memory_stats() + once = fn() + assert once is not None + torch.cuda.synchronize() + measured[name] = (None, None, torch.cuda.max_memory_allocated() - before) + del once + samples = {name: [] for name in calls} + order = list(calls) + if reverse: + order.reverse() + for index in range(20): + for name in order if index % 2 == 0 else order[::-1]: + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(5): + if mode == "graph": + measured[name][0].replay() + else: + calls[name]() + end.record() + end.synchronize() + samples[name].append(start.elapsed_time(end) / 5) + return { + name: { + "median_ms": statistics.median(values), + "samples_ms": values, + "peak_increment_bytes": measured[name][2], + } + for name, values in samples.items() + } + + +def provider_op(name): + if name == "native": + from rl_engine.backends.cuda.moe.nemotron_experts import NemotronExpertsOp + + return NemotronExpertsOp() + if name == "triton": + from rl_engine.backends.shared.triton.moe.nemotron_experts import TritonNemotronExpertsOp + + return TritonNemotronExpertsOp() + raise ValueError(f"unknown provider {name!r}: expected 'native' or 'triton'") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--tokens", type=int, nargs="+", default=[1, 16, 128, 513, 1024, 4096, 8192, 32768] + ) + parser.add_argument("--dtype", choices=list(DTYPES), default="bf16") + parser.add_argument("--distributions", nargs="+", default=["random", "concentrated"]) + parser.add_argument( + "--providers", nargs="+", choices=["native", "triton"], default=["native", "triton"] + ) + parser.add_argument( + "--baselines", + nargs="+", + choices=["eager", "grouped_mm"], + default=["eager"], + help="eager = per-expert FP32 torch.matmul (forward and backward); " + "grouped_mm = torch._grouped_mm, the cuBLAS/CUTLASS grouped-GEMM path " + "(forward only: its weight gradients need a grouped wgrad kernel)", + ) + parser.add_argument("--mode", choices=["graph", "eager"], default="graph") + parser.add_argument( + "--phases", + nargs="+", + choices=["forward", "forward_backward"], + default=["forward", "forward_backward"], + help="restrict the sweep; the forward-only phase is what a forward-only " + "baseline such as grouped_mm takes part in", + ) + parser.add_argument("--reverse", action="store_true") + parser.add_argument("--qualify-max-tokens", type=int, default=1024) + parser.add_argument( + "--native-backend", + choices=["auto", "sm90_v1", "sm90_v2"], + default="auto", + help="instruction path of the native provider; exports " + "RL_KERNEL_NEMOTRON_EXPERTS_BACKEND, which both paths answer identically", + ) + args = parser.parse_args() + if args.output.exists(): + parser.error("refusing to overwrite results") + os.environ["RL_KERNEL_NEMOTRON_EXPERTS_BACKEND"] = args.native_backend + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + torch.manual_seed(434) + dtype = DTYPES[args.dtype] + + providers = {} + for name in args.providers: + try: + providers[name] = provider_op(name) + except (RuntimeError, ImportError) as error: + print(f"provider {name} unavailable: {error}", flush=True) + if not providers: + parser.error("no provider available") + + result = { + "torch": torch.__version__, + "gpu": torch.cuda.get_device_name(), + "source_sha256": hashlib.sha256(Path(core.__file__).read_bytes()).hexdigest(), + "benchmark_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "provider_ids": {name: op.provider_id for name, op in providers.items()}, + "method": ( + "20 alternating paired samples, 5 executions per sample, median reported; " + f"mode={args.mode}; baselines={','.join(args.baselines)} where `baseline` is " + "per-expert FP32 torch.matmul and `grouped_mm` is torch._grouped_mm " + "(forward only), both with a weighted slot-ascending combine; cotangents " + "prepared outside timing; the public validation wrapper is outside both sides" + ), + "scope": ( + "grouped up/relu2/down + fixed-order combine + shared expert, forward and " + "forward+backward; no optimizer, no auxiliary loss, no communication" + ), + "baseline_notes": ( + "Transformer Engine 2.19 grouped GEMM is not installed, so the baseline is the " + "eager expert-by-expert formulation that actually ran" + ), + "precision": ( + "providers run the selected payload dtype with FP32 accumulation and the " + "contract's BF16 leaf rounding; the eager baseline widens every operand to " + "float32, so speedup_*_vs_baseline compares a BF16 operator with an FP32 " + "reference rather than two equal-precision GEMMs. grouped_mm runs BF16." + ), + "arguments": {**vars(args), "output": args.output.name}, + "rows": [], + } + + stream = torch.cuda.default_stream() if args.mode == "eager" else torch.cuda.Stream() + with torch.cuda.stream(stream): + for distribution in args.distributions: + for tokens in args.tokens: + case, meta = build(tokens, distribution, dtype=dtype) + cotangent = meta["cotangent"] + leaves = leaves_of(case) + calls = {} + if "eager" in args.baselines: + base_forward, base_backward = baseline_calls( + case, meta["spans"], leaves, cotangent + ) + calls["baseline"] = (base_forward, base_backward) + if "grouped_mm" in args.baselines: + try: + calls["grouped_mm"] = grouped_mm_calls(case) + except RuntimeError as error: + print(f"baseline grouped_mm unavailable: {error}", flush=True) + for name, op in providers.items(): + calls[name] = provider_calls(op, case, leaves, cotangent) + qualification = { + name: qualify(op, case, meta, args.qualify_max_tokens) + for name, op in providers.items() + } + diagnostics = { + "weights_bytes": meta["weights_bytes"], + "n_e": meta["n_e"], + "qualification": qualification, + } + for phase, index in (("forward", 0), ("forward_backward", 1)): + if phase not in args.phases: + continue + timed = { + name: closures[index] + for name, closures in calls.items() + if closures[index] is not None + } + measured = paired_measure(timed, args.reverse, args.mode) + row = { + "tokens": tokens, + "dtype": args.dtype, + "baseline_dtypes": {"eager": "float32", "grouped_mm": args.dtype}, + "distribution": distribution, + "phase": phase, + "reverse": args.reverse, + "providers": measured, + "diagnostics": diagnostics, + } + for name in measured: + if name == "baseline" or "baseline" not in measured: + continue + row[f"speedup_{name}_vs_baseline"] = ( + measured["baseline"]["median_ms"] / measured[name]["median_ms"] + ) + if "grouped_mm" in measured: + for name in providers: + if name in measured: + row[f"speedup_{name}_vs_grouped_mm"] = ( + measured["grouped_mm"]["median_ms"] + / measured[name]["median_ms"] + ) + result["rows"].append(row) + args.output.write_text(json.dumps(result, indent=2)) + summary = " ".join( + f"{name}={measured[name]['median_ms']:.4f}" for name in measured + ) + print(tokens, distribution, phase, summary, flush=True) + del calls, leaves, case, meta + torch.cuda.empty_cache() + print("TRAINING_BASELINE_DONE", flush=True) + + +if __name__ == "__main__": + main() diff --git a/build_tools/extensions.py b/build_tools/extensions.py index db079e828..5d45c9531 100644 --- a/build_tools/extensions.py +++ b/build_tools/extensions.py @@ -239,6 +239,9 @@ def get_extensions(): # This source contains NVIDIA PTX (cp.async, ldmatrix, and mma.sync). # The ROCm dispatcher falls back to PyTorch SDPA for this operator. cuda_sources.append("csrc/cuda/attention/prefix_shared_attention.cu") + # Grouped expert MLP (SM90 mma.sync / wgmma + TMA). Not hipified: ROCm + # uses the shared Triton provider, so a ROCm build must not see this file. + cuda_sources.append("csrc/cuda/moe/nemotron_experts.cu") nvcc_flags = ["-O3", "-Xfatbin", "-compress-all"] if envs.env_flag(envs.KERNEL_ALIGN_USE_FAST_MATH): diff --git a/csrc/bindings/ops.cpp b/csrc/bindings/ops.cpp index 0c7c3e7f4..e1ce07611 100644 --- a/csrc/bindings/ops.cpp +++ b/csrc/bindings/ops.cpp @@ -243,6 +243,56 @@ torch::Tensor det_gemm_fwd_rhs_transposed(torch::Tensor a, torch::Tensor bt); torch::Tensor det_gemm_da(torch::Tensor dc, torch::Tensor b); torch::Tensor det_gemm_db(torch::Tensor a, torch::Tensor dc); torch::Tensor det_gemm_db_transposed(torch::Tensor a, torch::Tensor dc); +#if !defined(USE_ROCM) +// Nemotron Nano grouped expert MLP + fixed-order weighted combine (SM90 CUDA). +// csrc/cuda/moe/nemotron_experts.cu is not hipified: ROCm uses the shared +// Triton provider instead, so these symbols exist only in CUDA builds. +bool nemotron_experts_compiled(); +const char* nemotron_experts_preferred_path(); +const char* nemotron_experts_last_path(); +std::tuple< + torch::Tensor, + torch::Tensor, + torch::Tensor, + torch::Tensor, + torch::Tensor, + c10::optional, + c10::optional, + c10::optional> +nemotron_experts_forward( + torch::Tensor packed, + c10::optional x, + torch::Tensor weights, + torch::Tensor permutation, + torch::Tensor offsets, + torch::Tensor w_up, + torch::Tensor w_down, + c10::optional w_sh1, + c10::optional w_sh2); +std::vector nemotron_experts_backward( + torch::Tensor grad_out, + torch::Tensor packed, + c10::optional x, + torch::Tensor weights, + torch::Tensor permutation, + torch::Tensor offsets, + torch::Tensor up, + torch::Tensor z, + torch::Tensor down, + torch::Tensor inverse, + torch::Tensor w_up, + torch::Tensor w_down, + c10::optional w_sh1, + c10::optional w_sh2, + c10::optional shared_up, + c10::optional shared_z, + c10::optional shared_down, + bool need_packed, + bool need_x, + bool need_weights, + bool need_weight_grad, + bool need_shared_weight_grad); +#endif // SiLU / SwiGLU Declarations (elementwise activation, general CUDA) torch::Tensor silu_forward_cuda(torch::Tensor x); torch::Tensor silu_backward_cuda(torch::Tensor dy, torch::Tensor x); @@ -717,6 +767,19 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { "det_gemm_db_transposed", &det_gemm_db_transposed, "Batch-invariant deterministic GEMM backward in canonical [N,K] layout"); +#if !defined(USE_ROCM) + // registry Nemotron grouped expert MLP + fixed-order weighted combine + m.def("nemotron_experts_compiled", &nemotron_experts_compiled, + "Whether the extension contains the SM90 Nemotron expert implementation"); + m.def("nemotron_experts_preferred_path", &nemotron_experts_preferred_path, + "Instruction path the next launch takes: sm90_v1, sm90_v2 or none"); + m.def("nemotron_experts_last_path", &nemotron_experts_last_path, + "Instruction path the last launch actually took, including the v1 shape fallback"); + m.def("nemotron_experts_forward", &nemotron_experts_forward, + "Grouped expert up/relu2/down + fixed-order combine + shared expert"); + m.def("nemotron_experts_backward", &nemotron_experts_backward, + "Grouped expert backward: dPacked, dX, dWeights, dW_up, dW_down, dW_shared"); +#endif // registry RMSNorm m.def("rmsnorm_forward", &rmsnorm_forward, "Batch-invariant RMSNorm forward CUDA"); m.def("rmsnorm_backward_dx", &rmsnorm_backward_dx, "Batch-invariant RMSNorm backward dx CUDA"); diff --git a/csrc/cuda/moe/nemotron_experts.cu b/csrc/cuda/moe/nemotron_experts.cu new file mode 100644 index 000000000..82b6d5675 --- /dev/null +++ b/csrc/cuda/moe/nemotron_experts.cu @@ -0,0 +1,1415 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2026 RL-Kernel Contributors +// +// csrc/cuda/moe/nemotron_experts.cu +// +// Nemotron Nano grouped expert MLP + fixed-order weighted combine (SM90). +// +// The grouped GEMM arithmetic is the det_gemm contract, copied deliberately: +// `csrc/cuda/gemm/det_gemm_kernel.cu` owns `det_gemm_sm90_kernel` and +// `docs/operators/det-gemm.md` documents it. Do not edit one without the +// other: `tests/backends/cuda/test_nemotron_experts_cuda.py` pins the up +// projection bitwise against `_C.det_gemm_fwd_rhs_transposed`. +// * BK = K_TREE_LEAF = 32 (one FP32 leaf), +// * each leaf is two m16n8k16 mma.sync calls into an FP32 accumulator and +// rounds once to BF16, +// * leaves merge with the mid-split tree (`mid_tree_merge_count`), BF16 nodes, +// * fixed ascending K order, one CTA per output tile, no split-K, no atomics. +// +// Differences from det_gemm, all in the mapping rather than in the arithmetic: +// * segments: grid.y = cdiv(rows, BM) + nseg; each CTA reads `offsets` in +// shared memory and resolves its (expert, row tile) itself, so the host +// never reads `offsets` (CUDA Graph capture forbids the sync); +// * stacked weights: `w_up [E,F,H]` and `w_down [E,H,F]` are one tensor map +// over [E*N,K]; expert e owns rows [e*N, (e+1)*N); +// * four operand modes (MODE_*) cover grouped forward, grouped backward and +// weight gradients; the operand that is stored transposed is read either +// with `ldmatrix.trans` (v1) or as an MN-major wgmma operand (v2); +// * the relu-square of the down projection is materialized by the up +// projection's epilogue as `Z = bf16(relu(U)**2)`, applied to bf16 values +// which is bitwise identical to the `__hmax2`/`__hmul2` fragment path and +// removes a per-tile operand transform from the down and dW_down stages. +// +// Two instruction paths satisfy the same arithmetic contract. Both are local +// to this file: the wgmma path needs det_gemm's descriptor, wgmma and swizzled +// TMA machinery, so it carries its own copy of those primitives rather than +// sharing them. +// * v1 `nemotron_experts_gemm`: 128x64 tiles, `mma.sync` + `ldmatrix`, one +// 32-wide K tile per TMA stage, the shape fallback; +// * v2 `nemotron_experts_gemm_v2`: 64x128 tiles, 128B TMA swizzle and +// `wgmma.mma_async` reading both operands from shared-memory descriptors, +// one producer warp plus a four-warp consumer warpgroup, bitwise identical +// to v1. +// `RL_KERNEL_NEMOTRON_EXPERTS_BACKEND=sm90_v1` forces the fallback for A/B +// measurement; `auto` (the default) picks v2 whenever the shape admits it. + +#include +#include +#include +#include +#include +#include + +#if defined(RL_KERNEL_ENABLE_SM90) +#include "../gemm/det_gemm_tma.cuh" +#endif + +namespace { + +using nv_bf16 = __nv_bfloat16; + +__host__ __device__ constexpr int cdiv(int a, int b) { return (a + b - 1) / b; } + +// Must match det_gemm's SM90 BK so a leaf is the same 32 K values. +constexpr int K_TREE_LEAF = 32; +// Merge-tree stack slots. The binary-counter merge holds one slot per level, so +// `TREE_DEPTH` covers `kd <= 1 << TREE_DEPTH` leaves; 14 covers the operator's +// whole domain (a segment can hold up to 6 * MAX_TOKENS = 393216 packed rows, +// i.e. kd = 12288) and `run_grouped` raises instead of overflowing if a caller +// ever exceeds it. det_gemm's own kernel keeps 12 because it caps K at 32768. +constexpr int TREE_DEPTH = 14; + +template +__device__ __forceinline__ output_t cast_output(float value); + +template <> +__device__ __forceinline__ nv_bf16 cast_output(float value) { + return __float2bfloat16(value); +} + +template <> +__device__ __forceinline__ float cast_output(float value) { + return value; +} + +// Fixed FP32 midpoint tree over reduction leaves (see det_gemm_kernel.cu). +__host__ __device__ __forceinline__ int mid_tree_merge_count(int leaf, int n) { + int lo = 0, hi = n, count = 0; + while (hi - lo > 1) { + const int mid = lo + (hi - lo) / 2; + if (leaf < mid) { + hi = mid; + count = 0; + } else { + lo = mid; + ++count; + } + } + return count; +} + +// --------------------------------------------------------------------------- +// Elementwise stages: permutation inverse, fixed-order combine, exit cast, +// route gradient and the FP32 weight-gradient dot products. +// --------------------------------------------------------------------------- + +__global__ void nemotron_experts_invert_kernel(int64_t* __restrict__ inverse, + const int64_t* __restrict__ permutation, + int routes) { + const int index = blockIdx.x * blockDim.x + threadIdx.x; + // `permutation` is a bijection (router contract), so with a valid table this + // scatter is race-free. `inverse` is zero-initialized by the caller and an + // out-of-range route id is ignored: a contract-violating permutation then + // yields a wrong result instead of an out-of-bounds write, and a duplicate + // leaves its slot at row 0 instead of exposing uninitialized memory. Neither + // check costs a host sync, so graph capture stays legal. + if (index < routes) { + const int64_t route = permutation[index]; + if (route >= 0 && route < routes) inverse[route] = index; + } +} + +// routed[t, j] = sum over slots ascending of weights[t, s] * Y[route_row(t, s), j]. +// One thread owns four FP32 columns of one token, so the slot order and the FP32 +// accumulation order are independent of the batch and of thread scheduling. +// The operator's only exit cast, fused with the combine: four FP32 columns per +// thread gather the six slots in ascending order, add the shared expert and round +// once, so no `routed` buffer is ever materialized. +__global__ void nemotron_experts_combine_kernel(nv_bf16* __restrict__ out, + const float* __restrict__ down, + const int64_t* __restrict__ inverse, + const float* __restrict__ weights, + const float* __restrict__ shared_down, + int width4) { + const int token = blockIdx.x; + const int segment = blockIdx.y * blockDim.x + threadIdx.x; + if (segment >= width4) return; + const int64_t slot_base = (int64_t)token * 6; + float4 total = make_float4(0.0f, 0.0f, 0.0f, 0.0f); +#pragma unroll + for (int slot = 0; slot < 6; ++slot) { + const float weight = weights[slot_base + slot]; + const int64_t row = inverse[slot_base + slot]; + const float4 value = + reinterpret_cast(down + row * (int64_t)width4 * 4)[segment]; + total.x += weight * value.x; + total.y += weight * value.y; + total.z += weight * value.z; + total.w += weight * value.w; + } + if (shared_down != nullptr) { + const float4 extra = + reinterpret_cast(shared_down + (int64_t)token * width4 * 4)[segment]; + total.x += extra.x; + total.y += extra.y; + total.z += extra.z; + total.w += extra.w; + } + nv_bf16* target = out + (int64_t)token * width4 * 4 + (int64_t)segment * 4; + target[0] = __float2bfloat16(total.x); + target[1] = __float2bfloat16(total.y); + target[2] = __float2bfloat16(total.z); + target[3] = __float2bfloat16(total.w); +} + +// dY[packed row] = bf16(weights[t, slot] * dOut[t]): one route weight per row of +// the packed payload, so the cast happens once per element. +__global__ void nemotron_experts_dy_kernel(nv_bf16* __restrict__ dy, + const nv_bf16* __restrict__ grad_out, + const int64_t* __restrict__ permutation, + const float* __restrict__ weights, + int width8) { + const int row = blockIdx.x; + const int segment = blockIdx.y * blockDim.x + threadIdx.x; + if (segment >= width8) return; + const int64_t original = permutation[row]; + const float weight = weights[original]; + const nv_bf16* source = grad_out + (original / 6) * (int64_t)width8 * 8 + (int64_t)segment * 8; + nv_bf16* target = dy + (int64_t)row * width8 * 8 + (int64_t)segment * 8; +#pragma unroll + for (int index = 0; index < 8; ++index) { + target[index] = __float2bfloat16(weight * __bfloat162float(source[index])); + } +} + +// One thread per (route, 32-wide leaf): sequential FP32 FMA inside the leaf, so +// the leaf value never depends on the block or warp shape. +__global__ void nemotron_experts_dweights_leaf_kernel(float* __restrict__ leaves, + const float* __restrict__ down, + const nv_bf16* __restrict__ grad_out, + const int64_t* __restrict__ inverse, + int routes, int leaf_count, int width) { + const int index = blockIdx.x * blockDim.x + threadIdx.x; + const int route = index / leaf_count; + if (route >= routes) return; + const int leaf = index % leaf_count; + const int token = route / 6; + const float* down_row = down + inverse[route] * (int64_t)width; + const nv_bf16* grad_row = grad_out + (int64_t)token * width; + const int lo = leaf * K_TREE_LEAF; + const int hi = lo + K_TREE_LEAF < width ? lo + K_TREE_LEAF : width; + float acc = 0.0f; + for (int column = lo; column < hi; ++column) + acc = fmaf(down_row[column], __bfloat162float(grad_row[column]), acc); + leaves[(int64_t)route * leaf_count + leaf] = acc; +} + +// Fixed FP32 midpoint tree over the leaf values; one thread per route. +__global__ void nemotron_experts_dweights_merge_kernel(float* __restrict__ dweights, + const float* __restrict__ leaves, + int leaf_count, int routes) { + const int route = blockIdx.x * blockDim.x + threadIdx.x; + if (route >= routes) return; + const float* values = leaves + (int64_t)route * leaf_count; + float stack[16]; + int depth = 0; + float value = 0.0f; + for (int leaf = 0; leaf < leaf_count; ++leaf) { + value = values[leaf]; + int merge = mid_tree_merge_count(leaf, leaf_count); + while (merge-- > 0) value = stack[--depth] + value; + if (leaf + 1 < leaf_count) { + stack[depth] = value; + ++depth; + } + } + dweights[route] = value; +} + +} // namespace + +#if defined(RL_KERNEL_ENABLE_SM90) + +namespace { + +constexpr int BM = 128, BN = 64, BK = 32; +static_assert(BK == K_TREE_LEAF, "grouped tile width must match the K-tree leaf"); +constexpr int WARPS = 4; +constexpr int WG_THREADS = WARPS * 32; +constexpr int STAGES = 2; +constexpr int MMA_M = 16, MMA_N = 8, MMA_K = 16; +constexpr int WARP_M = BM / WARPS; +constexpr int M_TILES = WARP_M / MMA_M; +constexpr int N_TILES = BN / MMA_N; +constexpr int K_TILES = BK / MMA_K; +constexpr int KK_GROUPS = BK / 32; + +// Operand modes. The A/B tile layouts are the only things that change; the +// leaf and tree arithmetic is shared by v1 and v2. +// +// MODE A operand B operand +// UP [rows,K] K contiguous [E*F, H] K contiguous +// DOWN [rows,F] K contiguous [E*H, F] K contiguous +// DX [rows,K] K contiguous [E*K, N] N contiguous +// WGRAD [rows,M] M contiguous [rows,N] N contiguous +// +// K contiguous is a K-major (Major::K) wgmma operand, read through +// `ldmatrix.trans` in v2's predecessors; M/N contiguous is Major::MN. +enum { + MODE_UP = 0, // grouped up projection, also the shared expert's + MODE_DOWN = 1, // grouped down projection, FP32 output for the combine + MODE_DX = 2, // dU = dY @ w_down^T, and dPacked = dU @ w_up^T + MODE_WGRAD = 3, // dW_up = dU^T @ packed, dW_down = dY^T @ Z +}; + +// Which CTA owns which (expert, row tile). The host never reads `offsets`, so +// CUDA Graph capture stays legal. Shared by both instruction paths. +struct Segment { + int expert; + int tile; + int64_t lo; + int64_t hi; +}; + +template +__device__ __forceinline__ Segment resolve_segment(const int64_t* offsets, int nseg, int M, + int rows) { + Segment out{-1, 0, 0, 0}; + if (offsets == nullptr) { + // Single segment [0, rows): the shared expert reuses this kernel verbatim. + const int tiles = cdiv(WGRAD ? M : rows, BM_); + if (blockIdx.y < (int64_t)tiles && rows > 0) { + out.expert = 0; + out.hi = rows; + out.tile = (int)blockIdx.y * BM_; + } + } else if constexpr (WGRAD) { + // Every expert owns cdiv(M,BM) output tiles, including empty segments: + // those write zeros instead of requiring a memset of the whole tensor. + const int tiles = cdiv(M, BM_); + if (blockIdx.y < (int64_t)tiles * nseg) { + out.expert = blockIdx.y / tiles; + out.tile = (blockIdx.y % tiles) * BM_; + out.lo = offsets[out.expert]; + out.hi = offsets[out.expert + 1]; + } + } else { + int remaining = blockIdx.y; + for (int e = 0; e < nseg; ++e) { + const int64_t begin = offsets[e], end = offsets[e + 1]; + const int tiles = (int)((end - begin + BM_ - 1) / BM_); + if (remaining < tiles) { + out.expert = e; + out.tile = remaining * BM_; + out.lo = begin; + out.hi = end; + break; + } + remaining -= tiles; + } + } + return out; +} + +__device__ __forceinline__ void ldmatrix_x4(uint32_t regs[4], uint32_t addr) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];" + : "=r"(regs[0]), "=r"(regs[1]), "=r"(regs[2]), "=r"(regs[3]) + : "r"(addr)); +} + +__device__ __forceinline__ void ldmatrix_x4_trans(uint32_t regs[4], uint32_t addr) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];" + : "=r"(regs[0]), "=r"(regs[1]), "=r"(regs[2]), "=r"(regs[3]) + : "r"(addr)); +} + +__device__ __forceinline__ void mma_m16n8k16(const uint32_t A[4], const uint32_t B[2], float D[4]) { + asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};" + : "=f"(D[0]), "=f"(D[1]), "=f"(D[2]), "=f"(D[3]) + : "r"(A[0]), "r"(A[1]), "r"(A[2]), "r"(A[3]), "r"(B[0]), "r"(B[1]), + "f"(D[0]), "f"(D[1]), "f"(D[2]), "f"(D[3])); +} + +// bf16(relu(x)**2) on a lane pair. A bf16 product is exact in FP32, so this +// equals rounding the FP32 square once, which is why the up projection can +// materialize the down projection's input without changing a bit. +__device__ __forceinline__ __nv_bfloat162 relu_square(__nv_bfloat162 value) { + const __nv_bfloat162 zero = __floats2bfloat162_rn(0.0f, 0.0f); + const __nv_bfloat162 clamped = __hmax2(value, zero); + return __hmul2(clamped, clamped); +} + +// One output column pair. Every mode writes BF16 pairs; MODE_UP additionally +// materializes Z = bf16(relu(U)**2), the down projection's operand, in two packed +// instructions that the down stage used to spend on every one of its operand +// tiles. `index` is the flat position of the pair's first element. +template +__device__ __forceinline__ void store_pair(output_t* __restrict__ C, nv_bf16* __restrict__ z, + const nv_bf16* __restrict__ aux, int64_t index, + float v0, float v1) { + if constexpr (EPI_RELU2_GRAD) { + // relu-square derivative: dU = dZ * 2 * relu(U), one rounding. + const __nv_bfloat162 saved = *reinterpret_cast(aux + index); + const float lo = __low2float(saved), hi = __high2float(saved); + v0 = v0 * (2.0f * (lo > 0.0f ? lo : 0.0f)); + v1 = v1 * (2.0f * (hi > 0.0f ? hi : 0.0f)); + } + if constexpr (std::is_same::value) { + const __nv_bfloat162 pair = __floats2bfloat162_rn(v0, v1); + *reinterpret_cast<__nv_bfloat162*>(C + index) = pair; + if constexpr (MODE == MODE_UP) + *reinterpret_cast<__nv_bfloat162*>(z + index) = relu_square(pair); + } else { + C[index + 0] = cast_output(v0); + C[index + 1] = cast_output(v1); + } +} + +template +__global__ void nemotron_experts_gemm(const __grid_constant__ CUtensorMap a_tmap, + const __grid_constant__ CUtensorMap b_tmap, + output_t* __restrict__ C, + nv_bf16* __restrict__ z, + const nv_bf16* __restrict__ aux, + const int64_t* __restrict__ offsets, + int nseg, int M, int N, int K, int rows) { + constexpr bool WGRAD = MODE == MODE_WGRAD; + constexpr bool TRANSPOSE_A = WGRAD; + constexpr bool TRANSPOSE_B = MODE == MODE_DX || WGRAD; + + const int tid = threadIdx.x; + const int warp = tid / 32; + const int lane = tid % 32; + const int col_base = blockIdx.x * BN; + + __shared__ Segment s_seg; + if (tid == 0) s_seg = resolve_segment(offsets, nseg, M, rows); + __syncthreads(); + if (s_seg.expert < 0) return; // Padding CTA above the real tile count. + + const int expert = s_seg.expert; + const int64_t seg_lo = s_seg.lo, seg_hi = s_seg.hi; + // Row start of the A operand (grouped modes) or of the output tile (dW). + const int64_t row_base = WGRAD ? (int64_t)s_seg.tile : seg_lo + s_seg.tile; + const int out_row_base = WGRAD ? s_seg.tile : (int)row_base; + const int64_t out_offset = WGRAD ? (int64_t)expert * M : 0; + const int tile_rows = M - s_seg.tile; + const int64_t segment_rows = seg_hi - row_base; + const int local_rows = WGRAD ? (tile_rows < BM ? tile_rows : BM) + : (int)(segment_rows < (int64_t)BM ? segment_rows : (int64_t)BM); + const int s_tile = s_seg.tile; + // Reduction depth: a fixed feature width, or the segment length for dW. + const int kd = WGRAD ? (int)((seg_hi - seg_lo + BK - 1) / BK) : (K / BK); + const int k_tail = (WGRAD && kd > 0) ? (int)(seg_hi - seg_lo - (int64_t)(kd - 1) * BK) : 0; + + extern __shared__ __align__(1024) char nemotron_smem[]; + nv_bf16* sA = reinterpret_cast(nemotron_smem); + nv_bf16* sB = reinterpret_cast(sA + STAGES * BM * BK); + int* mbar_base = reinterpret_cast(sB + STAGES * BN * BK); + + const uint32_t sA_base = static_cast(__cvta_generic_to_shared(sA)); + const uint32_t sB_base = static_cast(__cvta_generic_to_shared(sB)); + uint32_t mbar[STAGES]; +#pragma unroll + for (int s = 0; s < STAGES; ++s) + mbar[s] = static_cast(__cvta_generic_to_shared(mbar_base + 2 * s)); + + if (tid == 0) { +#pragma unroll + for (int s = 0; s < STAGES; ++s) det_gemm::mbar_init(mbar[s], 1); + asm volatile("fence.mbarrier_init.release.cluster;"); + } + __syncthreads(); + + const uint32_t tile_bytes = (BM * BK + BN * BK) * sizeof(nv_bf16); + + auto issue_load = [&](int k) { + const int buf = k % STAGES; + int a_x, a_y, b_x, b_y; + if constexpr (TRANSPOSE_A) { + a_x = s_tile; // column of the transposed payload == output row + a_y = (int)(seg_lo + (int64_t)k * BK); + } else { + a_x = k * BK; + a_y = (int)row_base; + } + if constexpr (MODE == MODE_DX) { + b_x = col_base; + b_y = (int)((int64_t)expert * K + (int64_t)k * BK); + } else if constexpr (WGRAD) { + b_x = col_base; + b_y = (int)(seg_lo + (int64_t)k * BK); + } else { + b_x = k * BK; + b_y = expert * N + col_base; + } + det_gemm::tma_2d_g2s(sA_base + buf * BM * BK * sizeof(nv_bf16), &a_tmap, a_x, a_y, mbar[buf]); + det_gemm::tma_2d_g2s(sB_base + buf * BN * BK * sizeof(nv_bf16), &b_tmap, b_x, b_y, mbar[buf]); + det_gemm::mbar_arrive_expect_tx(mbar[buf], tile_bytes); + }; + + int phase[STAGES]; +#pragma unroll + for (int s = 0; s < STAGES; ++s) phase[s] = 0; + + float tile_acc[M_TILES][N_TILES][4]; + __nv_bfloat162 tree_v[M_TILES][N_TILES][2]; + __nv_bfloat162 tree_stk[TREE_DEPTH][M_TILES][N_TILES][2]; + int sp = 0; + + // An empty expert has kd == 0, so the loop below never assigns tree_v. The + // weight-gradient grid deliberately covers every expert, so the tile is then + // stored as exact zeros instead of uninitialized stack. + const __nv_bfloat162 tree_zero = __floats2bfloat162_rn(0.0f, 0.0f); +#pragma unroll + for (int mi = 0; mi < M_TILES; ++mi) +#pragma unroll + for (int n = 0; n < N_TILES; ++n) +#pragma unroll + for (int i = 0; i < 2; ++i) tree_v[mi][n][i] = tree_zero; + + if (tid == 0) +#pragma unroll + for (int s = 0; s < STAGES - 1; ++s) + if (s < kd) issue_load(s); + + for (int k = 0; k < kd; ++k) { // fixed ascending tile order, NO split-K + const int buf = k % STAGES; + if (tid == 0 && k + (STAGES - 1) < kd) issue_load(k + (STAGES - 1)); + if (tid == 0) det_gemm::mbar_wait(mbar[buf], phase[buf]); + phase[buf] ^= 1; + __syncthreads(); + + const uint32_t sA_buf = sA_base + buf * BM * BK * sizeof(nv_bf16); + const uint32_t sB_buf = sB_base + buf * BN * BK * sizeof(nv_bf16); + + if constexpr (WGRAD) { + // The last reduction tile can read rows past this expert's segment; those + // rows belong to the next expert and must not join the reduction. + if (k + 1 == kd && k_tail < BK) { + nv_bf16* a_stage = sA + buf * BM * BK; + nv_bf16* b_stage = sB + buf * BN * BK; + for (int index = tid; index < (BK - k_tail) * BM; index += WG_THREADS) + a_stage[(int64_t)(k_tail + index / BM) * BM + index % BM] = __float2bfloat16(0.0f); + for (int index = tid; index < (BK - k_tail) * BN; index += WG_THREADS) + b_stage[(int64_t)(k_tail + index / BN) * BN + index % BN] = __float2bfloat16(0.0f); + __syncthreads(); + } + } + +#pragma unroll + for (int mi = 0; mi < M_TILES; ++mi) +#pragma unroll + for (int n = 0; n < N_TILES; ++n) + tile_acc[mi][n][0] = tile_acc[mi][n][1] = tile_acc[mi][n][2] = tile_acc[mi][n][3] = 0.0f; + + uint32_t A[M_TILES][K_TILES][4]; +#pragma unroll + for (int mi = 0; mi < M_TILES; ++mi) { +#pragma unroll + for (int kt = 0; kt < K_TILES; ++kt) { + if constexpr (TRANSPOSE_A) { + // Source tile is [BK,BM] (k-major): ldmatrix.trans builds the + // row-major A fragment. + const int k_row = kt * MMA_K + (lane / 16) * 8 + (lane % 8); + const int m_col = warp * WARP_M + mi * MMA_M + ((lane / 8) % 2) * 8; + ldmatrix_x4_trans(A[mi][kt], sA_buf + (k_row * BM + m_col) * sizeof(nv_bf16)); + } else { + const int row0 = warp * WARP_M + mi * MMA_M + (lane % 16); + ldmatrix_x4(A[mi][kt], + sA_buf + (row0 * BK + (lane / 16) * 8 + kt * MMA_K) * sizeof(nv_bf16)); + } + } + } + +#pragma unroll + for (int n = 0; n < N_TILES; ++n) { +#pragma unroll + for (int kk = 0; kk < KK_GROUPS; ++kk) { + uint32_t b4[4]; + if constexpr (TRANSPOSE_B) { + // Source tile is [BK,BN] (k-major). + const int k_row = kk * 32 + lane; + ldmatrix_x4_trans(b4, sB_buf + (k_row * BN + n * MMA_N) * sizeof(nv_bf16)); + } else { + ldmatrix_x4(b4, sB_buf + ((n * MMA_N + (lane % 8)) * BK + (lane / 8) * 8 + kk * 32) * + sizeof(nv_bf16)); + } + const uint32_t B0[2] = {b4[0], b4[1]}; + const uint32_t B1[2] = {b4[2], b4[3]}; +#pragma unroll + for (int mi = 0; mi < M_TILES; ++mi) { + mma_m16n8k16(A[mi][2 * kk + 0], B0, tile_acc[mi][n]); + mma_m16n8k16(A[mi][2 * kk + 1], B1, tile_acc[mi][n]); + } + } + } + __syncthreads(); + +#pragma unroll + for (int mi = 0; mi < M_TILES; ++mi) +#pragma unroll + for (int n = 0; n < N_TILES; ++n) +#pragma unroll + for (int i = 0; i < 2; ++i) + tree_v[mi][n][i] = + __floats2bfloat162_rn(tile_acc[mi][n][2 * i + 0], tile_acc[mi][n][2 * i + 1]); + + const int merge_count = mid_tree_merge_count(k, kd); + for (int merge = 0; merge < merge_count; ++merge) { +#pragma unroll + for (int mi = 0; mi < M_TILES; ++mi) +#pragma unroll + for (int n = 0; n < N_TILES; ++n) +#pragma unroll + for (int i = 0; i < 2; ++i) + tree_v[mi][n][i] = __hadd2(tree_stk[sp - 1][mi][n][i], tree_v[mi][n][i]); + --sp; + } + if (k + 1 < kd) { +#pragma unroll + for (int mi = 0; mi < M_TILES; ++mi) +#pragma unroll + for (int n = 0; n < N_TILES; ++n) +#pragma unroll + for (int i = 0; i < 2; ++i) tree_stk[sp][mi][n][i] = tree_v[mi][n][i]; + ++sp; + } + } + + // One column pair per (row, two adjacent columns). + auto emit = [&](int row_local, int col, float v0, float v1) { + const int64_t index = (out_offset + out_row_base + row_local) * (int64_t)N + col; + store_pair(C, z, aux, index, v0, v1); + }; + +#pragma unroll + for (int mi = 0; mi < M_TILES; ++mi) { + const int row = warp * WARP_M + mi * MMA_M + lane / 4; +#pragma unroll + for (int n = 0; n < N_TILES; ++n) { + const int col = col_base + n * MMA_N + (lane % 4) * 2; + if (col + 1 < N) { + if (row < local_rows) + emit(row, col, __low2float(tree_v[mi][n][0]), __high2float(tree_v[mi][n][0])); + if (row + 8 < local_rows) + emit(row + 8, col, __low2float(tree_v[mi][n][1]), __high2float(tree_v[mi][n][1])); + } + } + } +} + +template +void launch_grouped_v1(const nv_bf16* a_ptr, uint64_t a_height, uint64_t a_width, + const nv_bf16* b_ptr, uint64_t b_height, uint64_t b_width, + output_t* c_ptr, nv_bf16* z_ptr, const nv_bf16* aux_ptr, + const int64_t* offsets, int nseg, int M, int N, int K, int rows, + cudaStream_t stream) { + CUtensorMap a_tmap, b_tmap; + if constexpr (MODE == MODE_WGRAD) { + // A is the transposed payload: [rows, M] addressed in (k, m) tiles. + det_gemm::init_tmap_noswizzle(&a_tmap, a_ptr, a_height, a_width, BK, BM); + det_gemm::init_tmap_noswizzle(&b_tmap, b_ptr, b_height, b_width, BK, BN); + } else { + det_gemm::init_tmap_noswizzle(&a_tmap, a_ptr, a_height, a_width, BM, BK); + if constexpr (MODE == MODE_DX) + det_gemm::init_tmap_noswizzle(&b_tmap, b_ptr, b_height, b_width, BK, BN); + else + det_gemm::init_tmap_noswizzle(&b_tmap, b_ptr, b_height, b_width, BN, BK); + } + + // Grouped tiles: ceil(n_e/BM) sums to at most `rows`, so the tighter of the + // additive bound and `rows` avoids launching padding CTAs at decode batches + // (T=1 has 6 routes, not 129 candidate tiles). + const int64_t additive = (int64_t)cdiv(rows, BM) + nseg; + const int64_t grid_y = MODE == MODE_WGRAD + ? (int64_t)cdiv(M, BM) * nseg + : (additive < rows ? additive : (int64_t)rows); + if (grid_y <= 0) return; + const int smem = STAGES * (BM * BK + BN * BK) * sizeof(nv_bf16) + STAGES * 8; + if (smem > 48 * 1024) { + const cudaError_t attr_err = + cudaFuncSetAttribute(nemotron_experts_gemm, + cudaFuncAttributeMaxDynamicSharedMemorySize, smem); + TORCH_CHECK(attr_err == cudaSuccess, + "nemotron experts: cudaFuncSetAttribute failed: ", cudaGetErrorString(attr_err)); + } + dim3 grid(cdiv(N, BN), (unsigned)grid_y); + nemotron_experts_gemm<<>>( + a_tmap, b_tmap, c_ptr, z_ptr, aux_ptr, offsets, nseg, M, N, K, rows); +} + +// --------------------------------------------------------------------------- +// Version 2: the wgmma instruction path, applied per expert segment. +// +// The state machine is a warp-specialised wgmma pipeline: TMA with a 128-byte +// swizzle, one producer warp, a four-warp consumer warpgroup and three stages, +// so the K loop holds no __syncthreads. It is self-contained - the descriptor +// encoding, the wgmma wrappers, the swizzled tensor map and the extra barrier +// primitives below are this file's own. Grouped additions: +// * the CTA resolves its own (expert, row tile) exactly as v1 does; +// * either operand may be K-major or MN-major (`Major::K` / `Major::MN`), which +// selects the descriptor start offsets and the instruction's imm-trans flag; +// * a weight gradient's last reduction tile is zero-padded in shared memory, +// because a segment length is not a multiple of the 32-wide leaf. +// --------------------------------------------------------------------------- + +constexpr int V2_BM = 64, V2_BN = 128; +constexpr int V2_KT = 64; // one 128-byte inner box == two leaves +constexpr int V2_LEAVES = V2_KT / K_TREE_LEAF; +constexpr int V2_STAGES = 3; +constexpr int V2_WARPS = 4; // consumer warpgroup +constexpr int V2_THREADS = (V2_WARPS + 1) * 32; // plus the producer warp +constexpr int V2_GROUPS = V2_BN / 8; // wgmma column groups of eight +constexpr int V2_ACC = 4 * V2_GROUPS; // FP32 accumulators per thread +constexpr int V2_A_ELEMS = V2_BM * V2_KT; +constexpr int V2_B_ELEMS = V2_BN * V2_KT; +// MN-major blocks are 64 wide, so B's second block sits one 8 KB block further on. +constexpr int V2_BLOCK_ELEMS = 64 * V2_KT; +constexpr uint32_t V2_TILE_BYTES = (uint32_t)(V2_A_ELEMS + V2_B_ELEMS) * sizeof(nv_bf16); +constexpr uint32_t V2_MN_LEADING = V2_BLOCK_ELEMS * sizeof(nv_bf16) / 16; + +__device__ __forceinline__ uint32_t v2_smem_addr(const void* pointer) { + return static_cast(__cvta_generic_to_shared(pointer)); +} + +// wgmma matrix descriptor: 14-bit start address in 16-byte units, leading and +// stride byte offsets, 128-byte swizzle in the top field. For both operand +// majors the core matrices are eight 128-byte rows, so the stride offset is +// 1024 bytes; only the leading offset differs - it is unused for a K-major +// operand and the 8 KB MN-block step for a two-block MN-major B. +__device__ __forceinline__ uint64_t v2_desc(uint32_t smem_base, uint32_t byte_offset, + uint32_t leading) { + constexpr uint32_t kStride = 8 * 128 / 16; + const uint32_t address = smem_base + byte_offset; + return (static_cast(address >> 4) & 0x3FFFu) | + (static_cast(leading) << 16) | (static_cast(kStride) << 32) | + (uint64_t{1} << 62); +} + +#define NE_V2_ACC_OPERANDS \ + "{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,%11,%12,%13,%14,%15" \ + ",%16,%17,%18,%19,%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,%31" \ + ",%32,%33,%34,%35,%36,%37,%38,%39,%40,%41,%42,%43,%44,%45,%46,%47" \ + ",%48,%49,%50,%51,%52,%53,%54,%55,%56,%57,%58,%59,%60,%61,%62,%63}" + +#define NE_V2_ACC_CONSTRAINTS \ + "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]), "+f"(d[4]), "+f"(d[5]), \ + "+f"(d[6]), "+f"(d[7]), "+f"(d[8]), "+f"(d[9]), "+f"(d[10]), "+f"(d[11]), \ + "+f"(d[12]), "+f"(d[13]), "+f"(d[14]), "+f"(d[15]), "+f"(d[16]), "+f"(d[17]), \ + "+f"(d[18]), "+f"(d[19]), "+f"(d[20]), "+f"(d[21]), "+f"(d[22]), "+f"(d[23]), \ + "+f"(d[24]), "+f"(d[25]), "+f"(d[26]), "+f"(d[27]), "+f"(d[28]), "+f"(d[29]), \ + "+f"(d[30]), "+f"(d[31]), "+f"(d[32]), "+f"(d[33]), "+f"(d[34]), "+f"(d[35]), \ + "+f"(d[36]), "+f"(d[37]), "+f"(d[38]), "+f"(d[39]), "+f"(d[40]), "+f"(d[41]), \ + "+f"(d[42]), "+f"(d[43]), "+f"(d[44]), "+f"(d[45]), "+f"(d[46]), "+f"(d[47]), \ + "+f"(d[48]), "+f"(d[49]), "+f"(d[50]), "+f"(d[51]), "+f"(d[52]), "+f"(d[53]), \ + "+f"(d[54]), "+f"(d[55]), "+f"(d[56]), "+f"(d[57]), "+f"(d[58]), "+f"(d[59]), \ + "+f"(d[60]), "+f"(d[61]), "+f"(d[62]), "+f"(d[63]) + +// kAccumulate = false passes scale-d = 0, which overwrites the accumulator +// instead of accumulating into it, so the first k16 of a leaf needs no clear. +#define NE_V2_WGMMA(NAME, TRANS_A, TRANS_B) \ + template \ + __device__ __forceinline__ void NAME(float (&d)[V2_ACC], uint64_t a, uint64_t b) { \ + const uint32_t accumulate = kAccumulate ? 1u : 0u; \ + asm volatile( \ + "{\n.reg .pred p;\nsetp.ne.b32 p, %66, 0;\n" \ + "wgmma.mma_async.sync.aligned.m64n128k16.f32.bf16.bf16 " NE_V2_ACC_OPERANDS \ + ", %64, %65, p, 1, 1, " #TRANS_A ", " #TRANS_B ";\n}\n" \ + : NE_V2_ACC_CONSTRAINTS \ + : "l"(a), "l"(b), "r"(accumulate)); \ + } + +NE_V2_WGMMA(wgmma_kk, 0, 0) // A K-major, B K-major +NE_V2_WGMMA(wgmma_kmn, 0, 1) // A K-major, B MN-major +NE_V2_WGMMA(wgmma_mnmn, 1, 1) // both operands MN-major + +__device__ __forceinline__ void wgmma_commit() { + asm volatile("wgmma.commit_group.sync.aligned;"); +} +__device__ __forceinline__ void wgmma_wait_all() { + asm volatile("wgmma.wait_group.sync.aligned 0;"); +} +// Leaves at most one accumulator group in flight, so the previous leaf can be +// rounded and merged while the current one is still on the tensor pipe. +__device__ __forceinline__ void wgmma_wait_one() { + asm volatile("wgmma.wait_group.sync.aligned 1;"); +} +// Generic-to-async proxy fence: the shared-memory writes below must be visible +// to wgmma, which reads through the async proxy. +__device__ __forceinline__ void v2_fence_view_async_shared() { + asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); +} +// Plain barrier arrival: a consumer warp releasing its stage. +__device__ __forceinline__ void v2_mbar_arrive(uint32_t addr) { + asm volatile("mbarrier.arrive.release.cta.shared::cta.b64 _, [%0];" ::"r"(addr) : "memory"); +} +// Named barrier over the consumer warpgroup only (the producer warp never joins). +__device__ __forceinline__ void v2_consumer_barrier() { + asm volatile("barrier.sync 1, 128;" ::: "memory"); +} + +// One leaf of two k16 steps; `scale` picks whether the first step overwrites. +template +__device__ __forceinline__ void wgmma_step(float (&d)[V2_ACC], uint32_t a_stage, + uint32_t b_stage, uint32_t a_off, uint32_t b_off) { + const uint64_t a = v2_desc(a_stage, a_off, 1); + const uint64_t b = v2_desc(b_stage, b_off, B_MN ? V2_MN_LEADING : 1); + if constexpr (!A_MN && !B_MN) + wgmma_kk(d, a, b); + else if constexpr (!A_MN && B_MN) + wgmma_kmn(d, a, b); + else + wgmma_mnmn(d, a, b); +} + +// Zeroes the K rows of one shared-memory operand block that lie past the end of +// the expert's segment, so they cannot join the weight-gradient reduction. +__device__ __forceinline__ void v2_zero_rows(nv_bf16* tile, int first_row, int row_width, + int lane_offset, int stride) { + const int total = (V2_KT - first_row) * row_width; + for (int index = lane_offset; index < total; index += stride) + tile[(int64_t)(first_row + index / row_width) * row_width + index % row_width] = + __float2bfloat16(0.0f); +} + +template +__global__ void nemotron_experts_gemm_v2(const __grid_constant__ CUtensorMap a_tmap, + const __grid_constant__ CUtensorMap b_tmap, + output_t* __restrict__ C, nv_bf16* __restrict__ z, + const nv_bf16* __restrict__ aux, + const int64_t* __restrict__ offsets, + int nseg, int M, int N, int K, int rows) { + constexpr bool WGRAD = MODE == MODE_WGRAD; + constexpr bool A_MN = WGRAD; // A read in (k, m) tiles + constexpr bool B_MN = MODE == MODE_DX || WGRAD; // B read in (k, n) tiles + constexpr uint32_t kStepA = A_MN ? 128u : 2u; // bytes per K position + constexpr uint32_t kStepB = B_MN ? 128u : 2u; + + const int tid = threadIdx.x; + const int warp = tid / 32; + const int lane = tid % 32; + const int col_base = blockIdx.x * V2_BN; + + __shared__ Segment s_seg; + if (tid == 0) s_seg = resolve_segment(offsets, nseg, M, rows); + __syncthreads(); + if (s_seg.expert < 0) return; // Padding CTA above the real tile count. + + const int expert = s_seg.expert; + const int64_t seg_lo = s_seg.lo, seg_hi = s_seg.hi; + const int64_t row_base = WGRAD ? (int64_t)s_seg.tile : seg_lo + s_seg.tile; + const int out_row_base = WGRAD ? s_seg.tile : (int)row_base; + const int64_t out_offset = WGRAD ? (int64_t)expert * M : 0; + const int tile_rows = M - s_seg.tile; + const int64_t segment_rows = seg_hi - row_base; + const int local_rows = WGRAD ? (tile_rows < V2_BM ? tile_rows : V2_BM) + : (int)(segment_rows < (int64_t)V2_BM ? segment_rows + : (int64_t)V2_BM); + const int64_t seg_len = seg_hi - seg_lo; + const int kd = WGRAD ? (int)((seg_len + K_TREE_LEAF - 1) / K_TREE_LEAF) : (K / K_TREE_LEAF); + // Segment rows the last leaf must not see. + const int k_pad = WGRAD ? (int)((int64_t)kd * K_TREE_LEAF - seg_len) : 0; + const int chunks = cdiv(kd, V2_LEAVES); + // B's row coordinate: the expert's stacked [K, N] slice, or the segment rows. + const int64_t b_row_base = MODE == MODE_DX ? (int64_t)expert * K : seg_lo; + + extern __shared__ __align__(1024) char nemotron_smem[]; + nv_bf16* sA = reinterpret_cast(nemotron_smem); + nv_bf16* sB = sA + V2_STAGES * V2_A_ELEMS; + int64_t* barriers = reinterpret_cast(sB + V2_STAGES * V2_B_ELEMS); + uint32_t full[V2_STAGES], empty[V2_STAGES]; +#pragma unroll + for (int s = 0; s < V2_STAGES; ++s) { + full[s] = v2_smem_addr(barriers + 2 * s); + empty[s] = v2_smem_addr(barriers + 2 * s + 1); + } + if (tid == 0) { +#pragma unroll + for (int s = 0; s < V2_STAGES; ++s) { + det_gemm::mbar_init(full[s], 1); + det_gemm::mbar_init(empty[s], V2_WARPS); + } + asm volatile("fence.mbarrier_init.release.cluster;"); + } + __syncthreads(); + + if (warp == V2_WARPS) { // ---- producer warp ---- + if (lane == 0) { + for (int chunk = 0; chunk < chunks; ++chunk) { + const int buf = chunk % V2_STAGES; + if (chunk >= V2_STAGES) det_gemm::mbar_wait(empty[buf], ((chunk / V2_STAGES) + 1) & 1); + const uint32_t a_dst = v2_smem_addr(sA) + buf * V2_A_ELEMS * sizeof(nv_bf16); + const uint32_t b_dst = v2_smem_addr(sB) + buf * V2_B_ELEMS * sizeof(nv_bf16); + const int koff = chunk * V2_KT; + if constexpr (A_MN) { + det_gemm::tma_2d_g2s(a_dst, &a_tmap, s_seg.tile, (int)(seg_lo + koff), full[buf]); + } else { + det_gemm::tma_2d_g2s(a_dst, &a_tmap, koff, (int)row_base, full[buf]); + } + if constexpr (B_MN) { + det_gemm::tma_2d_g2s(b_dst, &b_tmap, col_base, (int)(b_row_base + koff), full[buf]); + det_gemm::tma_2d_g2s(b_dst + V2_BLOCK_ELEMS * sizeof(nv_bf16), &b_tmap, col_base + 64, + (int)(b_row_base + koff), full[buf]); + } else { + det_gemm::tma_2d_g2s(b_dst, &b_tmap, koff, expert * N + col_base, full[buf]); + } + det_gemm::mbar_arrive_expect_tx(full[buf], V2_TILE_BYTES); + } + } + return; + } + + // ---- consumer warpgroup ---- + float acc_a[V2_ACC]; + float acc_b[V2_ACC]; + __nv_bfloat162 tree_v[V2_ACC / 2]; + __nv_bfloat162 tree_stk[TREE_DEPTH][V2_ACC / 2]; + int sp = 0; + const __nv_bfloat162 tree_zero = __floats2bfloat162_rn(0.0f, 0.0f); + // An empty expert has kd == 0, so the loop below never assigns tree_v; the + // weight-gradient grid deliberately covers every expert, so such a tile is + // stored as exact zeros instead of uninitialized registers. +#pragma unroll + for (int i = 0; i < V2_ACC / 2; ++i) tree_v[i] = tree_zero; + + for (int chunk = 0; chunk < chunks; ++chunk) { + const int buf = chunk % V2_STAGES; + const int leaf_base = chunk * V2_LEAVES; + const int leaves_here = (chunk + 1 == chunks) ? (kd - leaf_base) : V2_LEAVES; + det_gemm::mbar_wait(full[buf], (chunk / V2_STAGES) & 1); // every consumer thread waits + + const uint32_t a_stage = v2_smem_addr(sA) + buf * V2_A_ELEMS * sizeof(nv_bf16); + const uint32_t b_stage = v2_smem_addr(sB) + buf * V2_B_ELEMS * sizeof(nv_bf16); + + if constexpr (WGRAD) { + if (k_pad > 0 && chunk + 1 == chunks) { + // The last chunk's K rows past the segment end hold the next expert's + // rows; zero them so they cannot join the reduction. + const int first = (int)(seg_len - (int64_t)chunk * V2_KT); + nv_bf16* b_tile = sB + buf * V2_B_ELEMS; + v2_zero_rows(sA + buf * V2_A_ELEMS, first, V2_BM, tid, V2_WARPS * 32); + v2_zero_rows(b_tile, first, 64, tid, V2_WARPS * 32); + v2_zero_rows(b_tile + V2_BLOCK_ELEMS, first, 64, tid, V2_WARPS * 32); + v2_fence_view_async_shared(); + v2_consumer_barrier(); + asm volatile("wgmma.fence.sync.aligned;" ::: "memory"); + } + } + + // Two static accumulators, used as a two-leaf group: while leaf 0 is rounded + // and merged the tensor pipe already executes leaf 1, so no drain sits in + // the K loop. A runtime-indexed array would push both into local memory. + auto issue_leaf = [&](float (&dst)[V2_ACC], int leaf) { + const uint32_t a_off = leaf * K_TREE_LEAF * kStepA; + const uint32_t b_off = leaf * K_TREE_LEAF * kStepB; + // Two k16 steps make one 32-wide FP32 leaf, exactly like two mma.sync in v1. + wgmma_step(dst, a_stage, b_stage, a_off, b_off); + wgmma_step(dst, a_stage, b_stage, a_off + 16 * kStepA, + b_off + 16 * kStepB); + wgmma_commit(); + }; + + auto finish_leaf = [&](float (&src)[V2_ACC], int index) { + // Leaf value: one BF16 rounding per output element, as v1. +#pragma unroll + for (int i = 0; i < V2_ACC / 2; ++i) + tree_v[i] = __floats2bfloat162_rn(src[2 * i + 0], src[2 * i + 1]); + const int merges = mid_tree_merge_count(index, kd); + for (int merge = 0; merge < merges; ++merge) { +#pragma unroll + for (int i = 0; i < V2_ACC / 2; ++i) tree_v[i] = __hadd2(tree_stk[sp - 1][i], tree_v[i]); + --sp; + } + if (index + 1 < kd) { +#pragma unroll + for (int i = 0; i < V2_ACC / 2; ++i) tree_stk[sp][i] = tree_v[i]; + ++sp; + } + }; + + if (leaves_here == V2_LEAVES) { + issue_leaf(acc_a, 0); + issue_leaf(acc_b, 1); + wgmma_wait_one(); // leaf 0 done, leaf 1 still on the tensor pipe + finish_leaf(acc_a, leaf_base + 0); + wgmma_wait_all(); + finish_leaf(acc_b, leaf_base + 1); + } else { + issue_leaf(acc_a, 0); + wgmma_wait_all(); + finish_leaf(acc_a, leaf_base + 0); + } + if (lane == 0) v2_mbar_arrive(empty[buf]); // this warp is done with the stage + } + + // ---- epilogue: the wgmma accumulator matches v1's tree_v layout ---- + const int row0 = warp * 16 + lane / 4; + auto emit = [&](int row_local, int col, float v0, float v1) { + const int64_t index = (out_offset + out_row_base + row_local) * (int64_t)N + col; + store_pair(C, z, aux, index, v0, v1); + }; +#pragma unroll + for (int g = 0; g < V2_GROUPS; ++g) { + const int col = col_base + g * 8 + 2 * (lane % 4); + if (col + 1 < N) { + if (row0 < local_rows) + emit(row0, col, __low2float(tree_v[2 * g + 0]), __high2float(tree_v[2 * g + 0])); + if (row0 + 8 < local_rows) + emit(row0 + 8, col, __low2float(tree_v[2 * g + 1]), __high2float(tree_v[2 * g + 1])); + } + } +} + +// wgmma reads its operands through a 128-byte swizzle, so the v2 tensor maps +// differ from the v1 ones only in that field. The inner box is one swizzle row, +// i.e. at most 64 BF16 elements. +void nemotron_init_tmap_swizzle128(CUtensorMap* tmap, const nv_bf16* gmem, uint64_t height, + uint64_t width, uint32_t box_h, uint32_t box_w) { + TORCH_CHECK(box_w * sizeof(nv_bf16) <= 128, + "nemotron experts: swizzle128 inner box must fit 128 bytes"); + uint64_t size[2] = {width, height}; + uint64_t stride[1] = {width * sizeof(nv_bf16)}; + uint32_t box[2] = {box_w, box_h}; + uint32_t estride[2] = {1, 1}; + const CUresult res = cuTensorMapEncodeTiled( + tmap, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 2, const_cast(gmem), size, stride, box, + estride, CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_L2_PROMOTION_NONE, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + TORCH_CHECK(res == CUDA_SUCCESS, "nemotron experts: cuTensorMapEncodeTiled (swizzle128) failed"); +} + +template +void launch_grouped_v2(const nv_bf16* a_ptr, uint64_t a_height, uint64_t a_width, + const nv_bf16* b_ptr, uint64_t b_height, uint64_t b_width, + output_t* c_ptr, nv_bf16* z_ptr, const nv_bf16* aux_ptr, + const int64_t* offsets, int nseg, int M, int N, int K, int rows, + cudaStream_t stream) { + constexpr bool WGRAD = MODE == MODE_WGRAD; + CUtensorMap a_tmap, b_tmap; + if constexpr (WGRAD) { + // A is the transposed payload: [rows, M] addressed in (k, m) tiles. + nemotron_init_tmap_swizzle128(&a_tmap, a_ptr, a_height, a_width, V2_KT, V2_BM); + nemotron_init_tmap_swizzle128(&b_tmap, b_ptr, b_height, b_width, V2_KT, 64); + } else { + nemotron_init_tmap_swizzle128(&a_tmap, a_ptr, a_height, a_width, V2_BM, V2_KT); + if constexpr (MODE == MODE_DX) + nemotron_init_tmap_swizzle128(&b_tmap, b_ptr, b_height, b_width, V2_KT, 64); + else + nemotron_init_tmap_swizzle128(&b_tmap, b_ptr, b_height, b_width, V2_BN, V2_KT); + } + + const int64_t additive = (int64_t)cdiv(rows, V2_BM) + nseg; + const int64_t grid_y = WGRAD ? (int64_t)cdiv(M, V2_BM) * nseg + : (additive < rows ? additive : (int64_t)rows); + if (grid_y <= 0) return; + const int smem = + V2_STAGES * (V2_A_ELEMS + V2_B_ELEMS) * sizeof(nv_bf16) + V2_STAGES * 16; + // The 72 KB dynamic shared-memory request is above the 48 KB default, and the + // attribute is scoped to the (function, device) pair, not to the process: a + // one-shot static would leave a second GPU in the same process unconfigured + // and its first launch would fail with `invalid argument`. Issue it on every + // launch (it is a host-side call next to the launch itself) and check it, + // because a dropped failure would only surface as a confusing launch error. + const cudaError_t attr_err = + cudaFuncSetAttribute(nemotron_experts_gemm_v2, + cudaFuncAttributeMaxDynamicSharedMemorySize, smem); + TORCH_CHECK(attr_err == cudaSuccess, + "nemotron experts: cudaFuncSetAttribute failed: ", cudaGetErrorString(attr_err)); + dim3 grid(cdiv(N, V2_BN), (unsigned)grid_y); + nemotron_experts_gemm_v2<<>>( + a_tmap, b_tmap, c_ptr, z_ptr, aux_ptr, offsets, nseg, M, N, K, rows); +} + +// `RL_KERNEL_NEMOTRON_EXPERTS_BACKEND`: `auto` (default) runs the wgmma path, +// `sm90_v1` pins the mma.sync path so the two can be measured against each +// other. Both satisfy the same contract and are bitwise identical. The +// variable is read per launch rather than cached, so a test or benchmark can +// compare the two paths inside one process. +// The instruction path the last `run_grouped` launch actually took. +// `preferred_path` is the env-selected preference *before* the per-shape gate; +// this records the branch that ran, so a test can prove a parity comparison +// executed two different kernels rather than the same one twice. Thread-local +// because a launch is host-thread work. +thread_local const char* experts_last_path = "none"; + +bool experts_prefer_v2() { + const char* value = std::getenv("RL_KERNEL_NEMOTRON_EXPERTS_BACKEND"); + if (value == nullptr) return true; + const std::string text(value); + if (text.empty() || text == "auto" || text == "sm90_v2" || text == "v2") return true; + TORCH_CHECK(text == "sm90_v1" || text == "v1", + "RL_KERNEL_NEMOTRON_EXPERTS_BACKEND must be auto, sm90_v1 or sm90_v2"); + return false; +} + +template +void run_grouped(const nv_bf16* a_ptr, uint64_t a_height, uint64_t a_width, + const nv_bf16* b_ptr, uint64_t b_height, uint64_t b_width, output_t* c_ptr, + nv_bf16* z_ptr, const nv_bf16* aux_ptr, const int64_t* offsets, int nseg, int M, + int N, int K, int rows, cudaStream_t stream) { + // The merge stack holds one slot per level, so fail closed rather than write + // past `tree_stk` when the reduction is deeper than the tree can cover. A + // weight gradient reconciles one leaf per 32 rows of an expert segment; every + // other mode reduces over K, which is independent of the row count (a wide + // hand-built call with a tiny batch must fail here, not overflow the stack). + // The weight-gradient bound uses the row count, not `offsets`, so the host + // still never reads the routing table. + const int64_t kd = MODE == MODE_WGRAD ? cdiv(rows, K_TREE_LEAF) : cdiv(K, K_TREE_LEAF); + TORCH_CHECK(kd <= (1 << TREE_DEPTH), + "nemotron experts: reduction depth exceeds the merge tree (", kd, " > ", + (1 << TREE_DEPTH), " leaves)"); + // v2 handles every geometry the operator accepts (its weight-gradient K is + // padded); the width check below only guards a hand-built call. + const bool v2_shape = N % 64 == 0 && M > 0 && (MODE == MODE_WGRAD || K % K_TREE_LEAF == 0); + if (experts_prefer_v2() && v2_shape) { + experts_last_path = "sm90_v2"; + launch_grouped_v2(a_ptr, a_height, a_width, b_ptr, b_height, + b_width, c_ptr, z_ptr, aux_ptr, offsets, nseg, + M, N, K, rows, stream); + return; + } + experts_last_path = "sm90_v1"; + launch_grouped_v1(a_ptr, a_height, a_width, b_ptr, b_height, + b_width, c_ptr, z_ptr, aux_ptr, offsets, nseg, + M, N, K, rows, stream); +} + +} // namespace + +#endif // RL_KERNEL_ENABLE_SM90 + +bool nemotron_experts_compiled() { +#if defined(RL_KERNEL_ENABLE_SM90) + return true; +#else + return false; +#endif +} + +// Which instruction path the next launch takes: the env-selected preference, +// before the per-shape gate. Provenance for route reporting and for the test +// that compares the two paths; "none" means the operator is not compiled. +const char* nemotron_experts_preferred_path() { +#if !defined(RL_KERNEL_ENABLE_SM90) + return "none"; +#else + return experts_prefer_v2() ? "sm90_v2" : "sm90_v1"; +#endif +} + +// The instruction path the last launch actually took, including the per-shape +// fallback to v1 that `preferred_path` does not model; "none" before the first +// launch or when the operator is not compiled. +const char* nemotron_experts_last_path() { +#if !defined(RL_KERNEL_ENABLE_SM90) + return "none"; +#else + return experts_last_path; +#endif +} + +namespace { + +const nv_bf16* bf16_ptr(const torch::Tensor& tensor) { + return reinterpret_cast(tensor.data_ptr()); +} +nv_bf16* bf16_mut(torch::Tensor& tensor) { + return reinterpret_cast(tensor.data_ptr()); +} + +// cuTensorMapEncodeTiled requires a 16-byte aligned global base; a contiguous +// view can miss that even when its values are right. +torch::Tensor aligned16(const torch::Tensor& tensor) { + if (reinterpret_cast(tensor.data_ptr()) % 16 == 0) return tensor; + return tensor.clone(); +} + +void check_bf16(const torch::Tensor& tensor, const char* name) { + TORCH_CHECK(tensor.is_cuda(), name, " must be CUDA"); + TORCH_CHECK(tensor.scalar_type() == torch::kBFloat16, name, " must be bfloat16"); + TORCH_CHECK(tensor.is_contiguous(), name, " must be contiguous"); +} + +void check_integer(const torch::Tensor& tensor, const char* name) { + TORCH_CHECK(tensor.is_cuda(), name, " must be CUDA"); + TORCH_CHECK(tensor.scalar_type() == torch::kInt64, name, " must be int64"); + TORCH_CHECK(tensor.is_contiguous(), name, " must be contiguous"); +} + +} // namespace + +std::tuple, c10::optional, + c10::optional> +nemotron_experts_forward(torch::Tensor packed, c10::optional x, + torch::Tensor weights, torch::Tensor permutation, + torch::Tensor offsets, torch::Tensor w_up, torch::Tensor w_down, + c10::optional w_sh1, + c10::optional w_sh2) { +#if !defined(RL_KERNEL_ENABLE_SM90) + TORCH_CHECK(false, "nemotron experts require an SM90 build (KERNEL_ALIGN_DET_GEMM_SM90=1)"); +#else + check_bf16(packed, "packed"); + check_bf16(w_up, "w_up"); + check_bf16(w_down, "w_down"); + check_integer(permutation, "permutation"); + check_integer(offsets, "offsets"); + TORCH_CHECK(weights.is_cuda() && weights.is_contiguous() && + weights.scalar_type() == torch::kFloat32, + "weights must be contiguous float32"); + TORCH_CHECK(packed.dim() == 2 && packed.size(0) % 6 == 0, "packed must be [6*tokens, hidden]"); + TORCH_CHECK(w_up.dim() == 3 && w_down.dim() == 3, "stacked weights must be rank three"); + const int rows = packed.size(0); + const int width = packed.size(1); + const int intermediate = w_up.size(1); + const int experts = w_up.size(0); + const int tokens = rows / 6; + // 64 is the TMA inner-box requirement. A width that is not a multiple of BN + // lets a B tile cross into the next expert's stacked rows, which only feeds + // output columns the store masks drop, so the schedule may exceed the width. + TORCH_CHECK(width % 64 == 0 && intermediate % 64 == 0, "hidden widths must be 64 aligned"); + TORCH_CHECK(width % 4 == 0, "hidden width must be vector aligned"); + TORCH_CHECK(w_up.size(2) == width && w_down.size(1) == width && w_down.size(2) == intermediate, + "stacked weight geometry must match the payload"); + TORCH_CHECK(w_down.size(0) == experts, "up and down weights must cover the same experts"); + TORCH_CHECK(offsets.numel() == experts + 1, "offsets must hold experts+1 prefix sums"); + TORCH_CHECK(permutation.numel() == rows, "permutation must cover every packed row"); + // Rank two is part of the contract, not just the element count: backward + // returns the route gradient as [tokens, 6], so a flat [rows] input would make + // autograd reject that gradient at accumulation time. + TORCH_CHECK(weights.dim() == 2 && weights.size(0) == tokens && weights.size(1) == 6, + "weights must be [tokens, 6] float32"); + TORCH_CHECK(w_sh1.has_value() == w_sh2.has_value(), "shared expert weights come as a pair"); + + const bool shared = w_sh1.has_value(); + torch::Tensor shared_x, shared_w1, shared_w2, shared_up, shared_z, shared_down; + int shared_width = 0; + if (shared) { + shared_w1 = aligned16(*w_sh1); + shared_w2 = aligned16(*w_sh2); + check_bf16(shared_w1, "w_sh1"); + check_bf16(shared_w2, "w_sh2"); + TORCH_CHECK(x.has_value(), "shared expert requires the token payload x"); + shared_x = aligned16(*x); + check_bf16(shared_x, "x"); + TORCH_CHECK(shared_w1.dim() == 2 && shared_w2.dim() == 2, "shared weights must be rank two"); + TORCH_CHECK(shared_w1.size(1) == width && shared_w2.size(0) == width, + "shared weight hidden widths must agree"); + TORCH_CHECK(shared_x.sizes() == torch::IntArrayRef({tokens, width}), + "x must be [tokens, hidden]"); + shared_width = shared_w1.size(0); + TORCH_CHECK(shared_w2.size(1) == shared_width, "shared intermediate widths must agree"); + TORCH_CHECK(shared_width % BN == 0, "shared intermediate width must be 64 aligned"); + } + + auto options = packed.options(); + auto float_options = options.dtype(torch::kFloat32); + auto out = torch::empty({tokens, width}, options); + auto up = torch::empty({rows, intermediate}, options); + auto z = torch::empty({rows, intermediate}, options); + auto down = torch::empty({rows, width}, float_options); + // Scatter-filled from `permutation`; zero so a non-bijective table cannot + // leave uninitialized row ids for the combine kernel to dereference. + auto inverse = torch::zeros({rows}, options.dtype(torch::kInt64)); + const c10::optional none; + if (tokens == 0) return {out, up, z, down, inverse, none, none, none}; + + auto packed_data_t = aligned16(packed); + auto up_weight_t = aligned16(w_up); + auto down_weight_t = aligned16(w_down); + const nv_bf16* packed_data = bf16_ptr(packed_data_t); + const nv_bf16* up_weight = bf16_ptr(up_weight_t); + const nv_bf16* down_weight = bf16_ptr(down_weight_t); + auto stream = at::cuda::getCurrentCUDAStream(); + + nemotron_experts_invert_kernel<<>>( + inverse.data_ptr(), permutation.data_ptr(), rows); + + run_grouped( + packed_data, rows, width, up_weight, (uint64_t)experts * intermediate, width, bf16_mut(up), + bf16_mut(z), nullptr, offsets.data_ptr(), experts, rows, intermediate, width, rows, + stream); + run_grouped( + bf16_ptr(z), rows, intermediate, down_weight, (uint64_t)experts * width, intermediate, + down.data_ptr(), nullptr, nullptr, offsets.data_ptr(), experts, rows, width, + intermediate, rows, stream); + + if (shared) { + shared_up = torch::empty({tokens, shared_width}, options); + shared_z = torch::empty({tokens, shared_width}, options); + shared_down = torch::empty({tokens, width}, float_options); + run_grouped( + bf16_ptr(shared_x), tokens, width, bf16_ptr(shared_w1), shared_width, width, + bf16_mut(shared_up), bf16_mut(shared_z), nullptr, nullptr, 1, tokens, shared_width, width, + tokens, stream); + run_grouped( + bf16_ptr(shared_z), tokens, shared_width, bf16_ptr(shared_w2), width, shared_width, + shared_down.data_ptr(), nullptr, nullptr, nullptr, 1, tokens, width, shared_width, + tokens, stream); + } + + // Combine, shared add and the single exit cast in one pass over the tokens. + const int width4 = width / 4; + nemotron_experts_combine_kernel<<>>( + bf16_mut(out), down.data_ptr(), inverse.data_ptr(), + weights.data_ptr(), shared ? shared_down.data_ptr() : nullptr, width4); + + if (!shared) return {out, up, z, down, inverse, none, none, none}; + return {out, up, z, down, inverse, shared_up, shared_z, shared_down}; +#endif +} + +std::vector nemotron_experts_backward( + torch::Tensor grad_out, torch::Tensor packed, c10::optional x, + torch::Tensor weights, torch::Tensor permutation, torch::Tensor offsets, torch::Tensor up, + torch::Tensor z, torch::Tensor down, torch::Tensor inverse, torch::Tensor w_up, + torch::Tensor w_down, c10::optional w_sh1, c10::optional w_sh2, + c10::optional shared_up, c10::optional shared_z, + c10::optional shared_down, bool need_packed, bool need_x, bool need_weights, + bool need_weight_grad, bool need_shared_weight_grad) { +#if !defined(RL_KERNEL_ENABLE_SM90) + TORCH_CHECK(false, "nemotron experts require an SM90 build (KERNEL_ALIGN_DET_GEMM_SM90=1)"); +#else + check_bf16(grad_out, "grad_out"); + check_bf16(packed, "packed"); + check_bf16(up, "up"); + check_bf16(z, "z"); + check_integer(permutation, "permutation"); + check_integer(offsets, "offsets"); + const int rows = packed.size(0); + const int width = packed.size(1); + const int intermediate = w_up.size(1); + const int experts = w_up.size(0); + const int tokens = rows / 6; + const bool shared = w_sh1.has_value(); + TORCH_CHECK(grad_out.sizes() == torch::IntArrayRef({tokens, width}), + "grad_out must be [tokens, hidden]"); + TORCH_CHECK(down.scalar_type() == torch::kFloat32 && down.is_contiguous() && down.is_cuda(), + "saved down must be contiguous float32"); + + auto options = packed.options(); + auto float_options = options.dtype(torch::kFloat32); + torch::Tensor empty; + torch::Tensor d_packed, d_x, d_weights, d_w_up, d_w_down, d_w_sh1, d_w_sh2; + if (tokens == 0) { + // An expert-parallel rank may hold no tokens. Every returned gradient is + // still shaped like its input: empty rows give empty payload gradients, and + // a weight gradient is exactly zero rather than absent, so an optimizer step + // and an all-reduce stay well defined. + const bool shared_grad = shared && need_shared_weight_grad; + return { + need_packed ? torch::zeros({rows, width}, options) : empty, + need_x ? torch::zeros({tokens, width}, options) : empty, + need_weights ? torch::zeros({tokens, 6}, float_options) : empty, + need_weight_grad ? torch::zeros_like(w_up) : empty, + need_weight_grad ? torch::zeros_like(w_down) : empty, + shared_grad ? torch::zeros_like(*w_sh1) : empty, + shared_grad ? torch::zeros_like(*w_sh2) : empty, + }; + } + if (!need_packed && !need_x && !need_weights && !need_weight_grad && + !(shared && need_shared_weight_grad)) + return {empty, empty, empty, empty, empty, empty, empty}; + + auto packed_data_t = aligned16(packed); + auto up_weight_t = aligned16(w_up); + auto down_weight_t = aligned16(w_down); + const nv_bf16* packed_data = bf16_ptr(packed_data_t); + const nv_bf16* up_weight = bf16_ptr(up_weight_t); + const nv_bf16* down_weight = bf16_ptr(down_weight_t); + const int64_t* route_offsets = offsets.data_ptr(); + auto stream = at::cuda::getCurrentCUDAStream(); + const int width8 = width / 8; + + auto dy = torch::empty({rows, width}, options); + nemotron_experts_dy_kernel<<>>( + bf16_mut(dy), bf16_ptr(grad_out), permutation.data_ptr(), + weights.data_ptr(), width8); + + if (need_weights) { + d_weights = torch::empty({tokens, 6}, float_options); + const int leaf_count = cdiv(width, K_TREE_LEAF); + auto leaves = torch::empty({rows, leaf_count}, float_options); + const int64_t total = (int64_t)rows * leaf_count; + nemotron_experts_dweights_leaf_kernel<<>>( + leaves.data_ptr(), down.data_ptr(), bf16_ptr(grad_out), + inverse.data_ptr(), rows, leaf_count, width); + nemotron_experts_dweights_merge_kernel<<>>( + d_weights.data_ptr(), leaves.data_ptr(), leaf_count, rows); + } + + // dU = dZ * 2*relu(U) is required by both dPacked and the up/down weight grads. + torch::Tensor du; + if (need_packed || need_weight_grad) { + du = torch::empty({rows, intermediate}, options); + run_grouped( + bf16_ptr(dy), rows, width, down_weight, (uint64_t)experts * width, intermediate, + bf16_mut(du), nullptr, bf16_ptr(up), route_offsets, experts, rows, intermediate, width, + rows, stream); + } + if (need_packed) { + d_packed = torch::empty({rows, width}, options); + run_grouped( + bf16_ptr(du), rows, intermediate, up_weight, (uint64_t)experts * intermediate, width, + bf16_mut(d_packed), nullptr, nullptr, route_offsets, experts, rows, width, intermediate, + rows, stream); + } + if (need_weight_grad) { + d_w_up = torch::empty_like(w_up); + d_w_down = torch::empty_like(w_down); + // dW_up[e] = dU_e^T @ packed_e, dW_down[e] = dY_e^T @ Z_e. + run_grouped( + bf16_ptr(du), rows, intermediate, packed_data, rows, width, bf16_mut(d_w_up), nullptr, + nullptr, route_offsets, experts, intermediate, width, 0, rows, stream); + run_grouped( + bf16_ptr(dy), rows, width, bf16_ptr(z), rows, intermediate, bf16_mut(d_w_down), nullptr, + nullptr, route_offsets, experts, width, intermediate, 0, rows, stream); + } + + if (shared) { + auto shared_w1 = aligned16(*w_sh1); + auto shared_w2 = aligned16(*w_sh2); + const int shared_width = shared_w1.size(0); + // `grad_out` becomes a TMA base for the shared dX/dW projections, so it + // needs the same 16-byte alignment `aligned16` guarantees everywhere else. + auto shared_grad_t = aligned16(grad_out); + const nv_bf16* shared_grad = bf16_ptr(shared_grad_t); + const nv_bf16* saved_shared_up = bf16_ptr(*shared_up); + torch::Tensor d_shared_up; + if (need_x || need_shared_weight_grad) { + d_shared_up = torch::empty({tokens, shared_width}, options); + run_grouped( + shared_grad, tokens, width, bf16_ptr(shared_w2), width, shared_width, + bf16_mut(d_shared_up), nullptr, saved_shared_up, nullptr, 1, tokens, shared_width, + width, tokens, stream); + } + if (need_x) { + d_x = torch::empty({tokens, width}, options); + run_grouped( + bf16_ptr(d_shared_up), tokens, shared_width, bf16_ptr(shared_w1), shared_width, width, + bf16_mut(d_x), nullptr, nullptr, nullptr, 1, tokens, width, shared_width, tokens, + stream); + } + if (need_shared_weight_grad) { + auto shared_x_t = aligned16(*x); + // `w_sh1`/`w_sh2` are rank two, so their gradients must be rank two as + // well: the WGRAD kernel writes `[M, N]` from offset zero, so a 2-D + // allocation holds exactly the same bytes. + d_w_sh1 = torch::empty({shared_width, width}, options); + d_w_sh2 = torch::empty({width, shared_width}, options); + run_grouped( + bf16_ptr(d_shared_up), tokens, shared_width, bf16_ptr(shared_x_t), tokens, width, + bf16_mut(d_w_sh1), nullptr, nullptr, nullptr, 1, shared_width, width, 0, tokens, + stream); + run_grouped( + shared_grad, tokens, width, bf16_ptr(*shared_z), tokens, shared_width, + bf16_mut(d_w_sh2), nullptr, nullptr, nullptr, 1, width, shared_width, 0, tokens, + stream); + } + } + + return {d_packed, d_x, d_weights, d_w_up, d_w_down, d_w_sh1, d_w_sh2}; +#endif +} diff --git a/docs/.nav.yml b/docs/.nav.yml index b26a5591d..cd1bb7679 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -24,6 +24,7 @@ nav: - operators/linear-logp-tp-test.md - operators/grpo-loss.md - operators/nemotron-router.md + - operators/nemotron-experts.md - operators/lm_head.md - operators/ratio-kl.md - operators/pack-and-pad.md diff --git a/docs/operators/README.md b/docs/operators/README.md index 5f9b9fa98..5dbb1e2e5 100644 --- a/docs/operators/README.md +++ b/docs/operators/README.md @@ -26,6 +26,7 @@ Every operator page should include: - [Fused Linear LogP TP Test Runbook](linear-logp-tp-test.md) - [GRPO Loss](grpo-loss.md) - [Nemotron Nano Router and Dispatch](nemotron-router.md) +- [Nemotron Nano Grouped Experts and Combine](nemotron-experts.md) - [RoPE](rope.md) - [LM Head](lm_head.md) - [Policy Ratio + KL Penalty](ratio-kl.md) diff --git a/docs/operators/nemotron-experts.md b/docs/operators/nemotron-experts.md new file mode 100644 index 000000000..2e3c3c031 --- /dev/null +++ b/docs/operators/nemotron-experts.md @@ -0,0 +1,601 @@ +# Nemotron Nano grouped experts and combine + +The `moe_experts_combine` row of [RFC #434](https://github.com/RL-Align/RL-Kernel/issues/434): +grouped up-projection GEMM with a relu-square epilogue, grouped down-projection, +a fixed-order weighted combine over the six route slots, the optional shared +expert and the operator's single output cast, forward and backward. The contract +is `nemotron-experts-sm90-v1` and is shared by two providers. + +Two implementations satisfy it: + +| Provider id | Column | Source | +| --- | --- | --- | +| `rlkernel.nemotron_experts.sm90.v1` | NVIDIA SM90 CUDA | `csrc/cuda/moe/nemotron_experts.cu` | +| `rlkernel.nemotron_experts.triton.v1` | CUDA and ROCm | `rl_engine/backends/shared/triton/moe/nemotron_experts.py` | + +Selection is the registry's, never a manual probe: the CUDA column lists the +native provider first and the Triton provider second, so a build without +`KERNEL_ALIGN_DET_GEMM_SM90=1` falls through to Triton instead of failing; the +ROCm column lists only the Triton provider, which is its sole implementation. +`cpu`, `npu`, `musa` and `mps` fail closed. `nemotron_experts_provider_id()` +reports the provider that would run, and the first call prints one +`[RL-Kernel][route] ... provider=` line when route reporting is enabled. + +The native provider carries two instruction paths behind one contract (see +[Instruction paths](#instruction-paths)): an `mma.sync` path and, by default, a +`wgmma` path. They are bitwise identical, so the provider id and the contract +version do not change with the path. + +The router row is separate: `nemotron_router_dispatch` produces `packed`, +`permutation` and `offsets`, this operator consumes them. + +## Interface + +```python +from rl_engine.ops.moe.nemotron_experts import nemotron_experts + +out = nemotron_experts(packed, x, weights, permutation, offsets, + w_up, w_down, w_sh1=None, w_sh2=None) +``` + +| Tensor | Shape | Dtype | Meaning | +| --- | --- | --- | --- | +| `packed` | `[6*T,2688]` | BF16 | Router payload, differentiable | +| `x` | `[T,2688]` | BF16 | Token payload for the shared expert; may be `None` | +| `weights` | `[T,6]` | FP32 | Router weights, slot ascending | +| `permutation` | `[6*T]` | int64 | Packed position to flat route slot | +| `offsets` | `[129]` | int64 | Exclusive expert prefix counts | +| `w_up` | `[128,1856,2688]` | BF16 | Stacked up-projection weights | +| `w_down` | `[128,2688,1856]` | BF16 | Stacked down-projection weights | +| `w_sh1` | `[3712,2688]` | BF16 | Shared up-projection; paired with `w_sh2` | +| `w_sh2` | `[2688,3712]` | BF16 | Shared down-projection | +| `out` | `[T,2688]` | BF16 | Combined expert output | + +Every input must be contiguous and on one CUDA/ROCm device; hidden and +intermediate widths must be 64 aligned. `0 <= T <= 65536`, which bounds every +32-bit index product. `w_sh1`/`w_sh2` come as a pair, and `x` is required only +when they are present. `permutation` and `offsets` are the router's bijection and +exclusive prefix sums: the hot path checks metadata only and does not re-validate +their values, so a hand-built dispatch must guarantee both. An empty batch +returns an empty output without launching a kernel. Validation raises +`ValueError`; an unavailable platform or provider raises `RuntimeError`. + +## Fixed arithmetic + +The grouped GEMMs are the [det_gemm](det-gemm.md) contract, copied on purpose: +`BK = K_TREE_LEAF = 32` is one FP32 leaf, each leaf is two `k16` MMA steps +accumulated in FP32 and rounded once to BF16, and leaves merge with the +canonical midpoint tree. No split-K, no atomics, no TF32, no autotuning. +`tests/backends/cuda/test_nemotron_experts_cuda.py` pins the up and down +projections bitwise against `_C.det_gemm_*`, and the same file pins `dPacked` +and both weight gradients. + +Weights are stacked as `[E,F,H]`/`[E,H,F]`, so one tensor map covers all experts +and expert `e` owns rows `[e*F, (e+1)*F)`. Two tensor maps are built per GEMM. + +| Stage | Output | Cast | +| --- | --- | --- | +| up | `U [6T,1856]` BF16 and `Z = bf16(relu(U)**2)` | leaf and tree rounding; `Z` in two packed instructions | +| down | `Y [6T,2688]` FP32 | BF16 tree value widened, one rounding | +| shared | `sh_out [T,2688]` FP32 | same up/down rules, `E=1` | +| combine + exit | `out = bf16(sum_slots w*Y + sh_out)` | the operator's only cast | +| dY | `bf16(fp32(w) * fp32(dOut))` | one rounding | +| dZ, dPacked | BF16 | leaf and tree rounding | +| dW | BF16 | grouped rows ascending | +| dWeights | `[T,6]` FP32 | 32-wide leaf, fixed FP32 tree | + +`relu(x)**2` is a BF16 operation on the up projection's already-rounded value. A +BF16 product is exact in FP32, so `bf16(relu(fp32(x))**2)` and `__hmul2` after +`__hmax2` produce identical bits. The up epilogue therefore materializes `Z` +itself - the down projection and `dW_down` consume it as an operand - instead of +each of them transforming every operand tile it loads. The transform is then +paid once per element rather than once per element per 128-wide output tile. +The relu-square derivative in the backward epilogue is `2 * relu(U)` multiplied +once and rounded once. + +## Instruction paths + +Both paths compute the arithmetic above and are **bitwise identical**; the choice +is an instruction-path decision, not a numerical one. + +| | v1 (`sm90_v1`) | v2 (`sm90_v2`, default) | +| --- | --- | --- | +| Tile | `BM=128`, `BN=64`, `BK=32` | `BM=64`, `BN=128`, `KT=64` (two leaves) | +| MMA | two `mma.sync.m16n8k16` per 32-wide leaf, per warp tile | two `wgmma.mma_async.m64n128k16` per 32-wide leaf, per CTA | +| Shared-memory layout | row-major, TMA swizzle none | TMA `SWIZZLE_128B`, wgmma descriptors | +| Operands | `ldmatrix` / `ldmatrix.trans` | read straight from shared memory by wgmma | +| Pipeline | 2 TMA stages, `__syncthreads` in the K loop | 3 stages, producer warp + 4-warp consumer warpgroup, mbarriers only | +| SASS | `HMMA 32`, `LDSM 12`, `REG 66-82`, `STACK 1552` | `HGMMA 6`, `LDSM 0`, `REG 135` (`170` dW), `STACK 1568` | +| Dynamic shared memory | 24.6 KB | 73.7 KB | +| Occupancy ceiling | 5 CTAs/SM by registers (31% of warp slots) | 3 CTAs/SM (`2` for dW) by registers and shared memory (23%, 16%) | + +`RL_KERNEL_NEMOTRON_EXPERTS_BACKEND=sm90_v1|sm90_v2|auto` selects the path +(`auto` is v2) and is read per launch, so one process can measure both. +`_C.nemotron_experts_preferred_path()` reports that selection, and +`_C.nemotron_experts_last_path()` reports the path the last launch actually took +(including the per-shape fallback to v1); the parity test asserts the executed +path around each run, so it cannot degrade into a kernel compared with itself. v2 is a +warp-specialised wgmma pipeline applied per expert segment, self-contained in +`csrc/cuda/moe/nemotron_experts.cu` (its descriptor encoding, wgmma wrappers, +swizzled tensor map and barrier primitives are that file's own), with three +grouped additions: + +* **Operand majors.** A wgmma operand is K-major when K is contiguous and + MN-major otherwise, and the kernel derives the descriptor start offsets and the + instruction's `imm-trans` flag from the mode: the up/down projections are + K-major on both sides, `DX` reads the stacked weights as an MN-major `[K,N]` + operand, and a weight gradient is MN-major on both sides (`[rows,M]` and + `[rows,N]`). An MN-major 128-wide B operand occupies two 64-wide blocks one + 8 KB block apart, which is the descriptor's leading byte offset; the stride + byte offset is 1024 B for both majors, because in both cases a core matrix is + eight 128-byte rows. +* **Segment resolution is unchanged.** The CTA still resolves its + `(expert, row tile)` from the 129 prefix sums in shared memory. +* **Weight-gradient tail padding.** A segment length is not a multiple of the + 32-wide leaf, so the last reduction tile zeroes the rows past the segment end in + shared memory before the wgmma reads it, with a generic-to-async proxy fence + and a warpgroup barrier. Rows past the segment belong to the next expert and + must not join the reduction. + +The Triton provider folds leaves in ascending order instead of merging them with +the midpoint tree. Its frontend rejects plain-Python helpers inside a kernel, and +`tl.static_range` rejects the list operations the stack algorithm needs, so a +trace-time stack is not expressible; the alternative, encoding the pop pattern as +eight nested static conditionals, buys a per-leaf association difference that the +contract does not claim. Leaf width, the single BF16 rounding per leaf and the +fixed ascending order are all preserved and every kernel folds the same way, so +batch invariance is unaffected; only the association of the leaf sums differs +from the native tree. `dWeights` folds 32-wide FP32 leaves the same way. + +* **Weight-gradient reduction depth is bounded.** A weight gradient reconciles one + 32-wide leaf per 32 rows of an expert segment, and the midpoint tree keeps one + merge slot per level, so `TREE_DEPTH = 14` covers `kd <= 16384` leaves. The + operator's own bound (`rows <= 6 * MAX_TOKENS = 393216`, i.e. `kd <= 12288`) is + inside it, and a hand-built call beyond it raises instead of writing past the + merge stack. The guard uses the row count, never `offsets`, so the host still + does not read the routing table and CUDA Graph capture stays legal. +* **Gradients are shaped like their inputs.** `dW_sh1`/`dW_sh2` are `[F,H]`/`[H,F]` + (not `[1,F,H]`), and the shared suite asserts the shape of every gradient + against its input. +* **A zero-token rank still returns shaped gradients.** An expert-parallel rank + can hold no tokens; `out` is then `[0,H]`, payload gradients are empty rows, and + every requested weight gradient is an exact zero of the weight's shape rather + than an absent tensor. Both providers implement this and the suite asserts it. + +## Ordering policy + +* **Experts ascend.** A CTA resolves its `(expert, row tile)` by walking the 129 + `offsets` in shared memory, so the host never reads `offsets` and CUDA Graph + capture stays legal. The grid is the upper bound `cdiv(rows,BM) + E`; surplus + CTAs return immediately. +* **Combine slots ascend.** For every token row the six route slots are summed + in ascending slot order into an FP32 accumulator, four columns per thread. +* **`dW` visits grouped rows ascending.** One pass per expert writes the whole + weight-gradient tile; empty experts write exact zeros, which is why no + full-tensor memset is needed. + +## Fail-closed gates + +A non-SM90 device (the native provider checks `packed.device` on every call, so +the verdict follows the device that will actually run, not the current device at +construction), a build without `KERNEL_ALIGN_DET_GEMM_SM90=1`, a dtype, rank or +contiguity mismatch, widths not 64-aligned, `tokens > 65536`, an `offsets` table +that is not `experts + 1` long, unpaired shared weights and a reduction depth +beyond the merge tree all raise. `permutation`/`offsets` *values* are trusted as +the router's contract rather than validated on device - a device-side check would +force a host synchronization and break graph capture - so a hand-built dispatch +must supply a bijection and its exclusive prefix sums. The combine's inverse +table is zero-initialized and its scatter ignores out-of-range route ids, so a +duplicate or out-of-range entry cannot make the combine dereference an +uninitialized or out-of-bounds `down` row; the other stages still trust the +table as the router contract. + +## Qualification boundary + +Within one provider, an identical token row and its upstream gradient keep the +same bits across batch sizes and positions, for `out`, `dPacked`, `dX` and +`dWeights`. `dW` repeats for an identical ordered token set; adding +independently computed microbatch `dW` is not a bitwise full-batch equivalence +claim. The two native instruction paths *are* bitwise equal to each other on +every output and gradient - they differ in how the tile is computed, not in how +it is reduced - which +`test_instruction_paths_agree_bitwise` asserts on the core geometry (including +per-expert segment lengths that are not multiples of the leaf) and on the +Nemotron geometry. The two providers do **not** claim bitwise equality: each is +qualified against FP64 on its own selected branch, exactly as the CUDA and +Triton `det_gemm` implementations coexist. + +Measured relative L2 against an unrounded FP64 pipeline, 17 tokens, shared expert +on, worst gradient per provider: + +| Geometry | native | Triton | native vs Triton | +| --- | --- | --- | --- | +| core (256/128, 8 experts) | 7.1e-3 | 5.4e-3 | 8.3e-3 | +| Nemotron (2688/1856, 128 experts) | 8.8e-3 | 1.0e-2 | 1.3e-2 | + +The cross-provider column stays near one BF16 ulp (7.8e-3): both providers round +every 32-wide leaf and cast once at the exit, but they associate the leaf sums +differently, and a BF16 output cannot resolve a smaller difference. Cross-compiler +equivalence, expert/sequence +parallelism and tensor parallelism are outside this operator. The ROCm column +was authored without ROCm hardware; it runs the same assertions as CUDA and +needs hardware re-qualification before it is quoted as verified. + +## Tests + +```bash +python -m pytest tests/reference/test_nemotron_experts_reference.py -q +python -m pytest tests/runtime/test_kernel_registry.py -q +python -m pytest tests/backends/cuda/test_nemotron_experts_cuda.py -q +python -m pytest tests/backends/rocm/test_nemotron_experts_rocm.py -q # ROCm host +``` + +The reference suite is CPU-only: an independent scalar FP64 oracle on toy +geometry, empty experts, the token-count bound, cast points, token permutation +and FP64 `gradcheck`. The provider suites compare against an FP64 emulation of +the same pipeline, assert raw-bit batch and relocation invariance, repeatability, +CUDA Graph replay, autocast invariance and geometry rejection. +`test_instruction_paths_agree_bitwise` runs both native instruction paths in one +process and compares every output and gradient with `torch.equal`. + +## Performance reproduction + +The baseline runs the same math expert by expert with `torch.matmul` in FP32 and a +weighted, slot-ascending combine - the obvious eager formulation. Its expert span +list is read once outside the timed region. The candidate rows are the provider +objects themselves, so the public validation wrapper is outside both sides of the +comparison. + +**The production grouped-GEMM path is measured for the forward, and it is 1.7-2.8x +faster than this operator.** `torch._grouped_mm` - the cuBLAS/CUTLASS grouped GEMM +a serving stack calls - is in the tables below as `grouped_mm`, measured in the +same paired run with the stacked weights pre-transposed outside the timed region +and the same slot-ascending combine. That gap is not an implementation defect and +is not addressed by any schedule change: the grouped GEMM keeps an FP32 +accumulator and casts once per output element, while the contract rounds and +tree-merges every 32-wide leaf, which is `K/32` extra SIMT passes per element. See +[Why v2 is the contract's ceiling](#why-v2-is-the-contracts-ceiling). + +What is **not** measured: Transformer Engine `GroupedLinear` (the repository pins +`transformer_engine==2.19.0`; it is not installed and does not build against this +container's CUDA 13.0 + PyTorch 2.11.0+cu130), vLLM's fused MoE (not installed), +and a production *training* baseline - `torch._grouped_mm` groups over token rows +only, so the weight gradients (`dW_up[e] = dU_eᵀ·packed_e`, whose reduction length +differs per expert) are not expressible with it, and the forward+backward tables +therefore still compare against the eager formulation. + +```bash +# forward: the two native paths, Triton, the eager formulation and the +# production grouped-GEMM path, in one paired run +PYTHONPATH=. python benchmarks/operators/moe/benchmark_nemotron_experts_training.py \ + --tokens 1 16 128 513 1024 4096 8192 --dtype bf16 \ + --providers native triton --baselines eager grouped_mm --phases forward \ + --distributions random concentrated --native-backend sm90_v2 \ + --output /tmp/experts-fwd-v2.json +# same sweep with the mma.sync path: +PYTHONPATH=. python benchmarks/operators/moe/benchmark_nemotron_experts_training.py \ + --tokens 1 16 128 513 1024 4096 8192 --dtype bf16 --providers native triton \ + --baselines eager grouped_mm --phases forward \ + --native-backend sm90_v1 --output /tmp/experts-fwd-v1.json +# forward+backward (the eager formulation only: a grouped weight gradient needs a +# grouped wgrad kernel, which torch._grouped_mm does not expose) +PYTHONPATH=. python benchmarks/operators/moe/benchmark_nemotron_experts_training.py \ + --tokens 1 16 128 513 1024 4096 8192 --dtype bf16 --providers native triton \ + --baselines eager --phases forward_backward --distributions random concentrated \ + --native-backend sm90_v2 --qualify-max-tokens 128 --output /tmp/experts-bwd-v2.json +``` + +`--baselines grouped_mm` is `torch._grouped_mm`, the cuBLAS/CUTLASS grouped GEMM a +serving stack calls: 2D operand plus a `[E,N,K]` stack and the group ends, with the +stacked weights materialized in the layout each projection needs outside the timed +region. It is the *production* comparison RFC #434 §4.1 asks for, it is +**forward only** (its weight gradients would need a grouped wgrad kernel), and it +does not implement the leaf-rounding contract, so it measures what the contract +costs rather than a better implementation of it. Transformer Engine +`GroupedLinear` (pinned 2.19.0) and vLLM's fused MoE are not installed in this +container and were not run. Forward and +forward+backward are measured separately: 20 alternating paired samples of five +executions, median reported, CUDA Graph replay, cotangents prepared outside +timing. `--reverse` repeats the sweep with the reversed initial provider order; +the two orders agree to a median 0.4% difference (worst sampled cell 2.8%), so the +tables below show the normal order only. The script refuses to overwrite an +existing output file. Every column of every table below comes from one sequential +session on an idle device at its boost clock (`clocks.sm` 1785 MHz of 1785); the +32768-token rows are a separate invocation because four candidates at that size +exceed the memory budget, and their baselines agree with the 1-8192 run to 1%. + +### Forward, random routing, median ms + +`grouped_mm` is the production path; `grouped_mm/v2` below 1 means the contract +kernel is slower, which it is everywhere in this direction. + +| T | baseline | grouped_mm | native v1 | native v2 | Triton | v1/v2 | grouped_mm/v2 | +| --- | --- | --- | --- | --- | --- | --- | --- | +| 1 | 0.477 | 0.209 | 0.782 | 0.358 | 0.473 | 2.18x | 0.58x | +| 16 | 4.277 | 0.737 | 3.558 | 1.416 | 1.818 | 2.51x | 0.52x | +| 128 | 10.088 | 1.147 | 5.925 | 2.315 | 3.106 | 2.56x | 0.50x | +| 513 | 11.823 | 1.328 | 6.268 | 2.480 | 3.390 | 2.53x | 0.54x | +| 1024 | 14.210 | 1.535 | 6.608 | 2.914 | 3.670 | 2.27x | 0.53x | +| 4096 | 29.604 | 3.587 | 15.200 | 9.850 | 8.967 | 1.54x | 0.36x | +| 8192 | 49.075 | 6.903 | 28.293 | 19.081 | 16.470 | 1.48x | 0.36x | +| 32768 | 164.884 | 27.742 | 107.390 | 72.175 | 65.414 | 1.49x | 0.38x | + +Under concentrated routing the same columns give `grouped_mm/v2` = 0.64 / 0.51 / +0.41 / 0.42 / 0.38 / 0.35 / 0.37 / 0.42, i.e. the production path is 1.6-2.9x +faster there too. + +### Forward + backward, random routing, median ms + +| T | baseline | native v1 | native v2 | Triton | v1/v2 | baseline/v2 | +| --- | --- | --- | --- | --- | --- | --- | +| 1 | 16.655 | 4.365 | 3.595 | 1.773 | 1.21x | 4.63x | +| 16 | 217.770 | 10.688 | 7.127 | 5.124 | 1.50x | 30.56x | +| 128 | 403.635 | 16.789 | 10.803 | 8.323 | 1.55x | 37.36x | +| 513 | 414.544 | 18.510 | 11.237 | 8.983 | 1.65x | 36.89x | +| 1024 | 425.932 | 21.360 | 12.624 | 10.647 | 1.69x | 33.74x | +| 4096 | 503.735 | 51.599 | 32.914 | 26.341 | 1.57x | 15.30x | +| 8192 | 602.836 | 95.971 | 58.188 | 49.231 | 1.65x | 10.36x | +| 32768 | 1201.012 | 357.091 | 215.282 | 184.260 | 1.66x | 5.58x | + +### Forward + backward, concentrated routing, median ms + +| T | baseline | native v1 | native v2 | Triton | v1/v2 | baseline/v2 | +| --- | --- | --- | --- | --- | --- | --- | +| 1 | 16.762 | 4.314 | 3.564 | 1.775 | 1.21x | 4.70x | +| 16 | 17.422 | 4.429 | 3.717 | 1.772 | 1.19x | 4.69x | +| 128 | 18.825 | 4.761 | 4.045 | 1.934 | 1.18x | 4.65x | +| 513 | 24.737 | 8.882 | 6.445 | 3.789 | 1.38x | 3.84x | +| 1024 | 31.542 | 12.987 | 9.314 | 5.679 | 1.39x | 3.39x | +| 4096 | 73.493 | 44.880 | 28.405 | 22.144 | 1.58x | 2.59x | +| 8192 | 135.217 | 89.930 | 55.296 | 45.252 | 1.63x | 2.45x | +| 32768 | 496.592 | 356.957 | 218.603 | 183.355 | 1.63x | 2.27x | + +### Peak memory + +Peak allocation increment over the shared weight stack, random routing, MiB +(`torch.cuda.max_memory_allocated` around one captured call, per candidate): + +| Phase | T | baseline | grouped_mm | native (both paths) | Triton | +| --- | --- | --- | --- | --- | --- | +| forward | 1 | 38 | 0.2 | 0.1 | 0.1 | +| forward | 1024 | 148 | 127 | 138 | 109 | +| forward | 8192 | 913 | 1010 | 1094 | 862 | +| forward+backward | 1 | 3878 | - | 2496 | 2496 | +| forward+backward | 1024 | 8775 | - | 2713 | 2684 | +| forward+backward | 8192 | 10613 | - | 4349 | 4117 | + +The forward+backward rows are dominated by the saved FP32 `down` (`[6T,2688]`) and +the BF16 `up` and `Z` (`[6T,1856]` each); `grouped_mm` keeps no saved state because +it is forward only. Both native paths allocate identically - both materialize `Z`, +which is 182 MiB of the 8192 forward+backward row. + +Reading the tables: + +* the wgmma path is 1.2-2.6x faster than the mma.sync path: 1.4-2.6x for forward, + 1.2-1.7x for forward+backward, with the smallest gain in the decode-sized + backward where launch and segment-resolution work dominates; +* forward+backward beats the eager baseline by 2.3-37.4x (v2) and 2.7-47x (Triton); + forward alone by 1.3-4.9x (v2) and 1.0-3.6x (Triton); +* the production grouped GEMM is 1.7-2.8x faster than this operator in the + forward (1.6-2.9x concentrated) - the price of the leaf-rounding contract, not + of the schedule. Triton, which also skips the per-leaf rounding, sits between + the two at 1.2-1.4x of the native path; +* forward at T=1 is not a regression: the `mma.sync` path's 0.782 ms against a + 0.477 ms eager baseline was 1.6x slower, the wgmma path is 0.358 ms, 1.33x + faster than the baseline and only 1.7x off `grouped_mm` (0.209 ms). + `BM=64` starts 6 CTAs instead of 3 on a six-route payload, and the narrower tile + is the only change that matters at that size; +* concentrated routing touches six experts, so every provider reads only non-empty + experts' weights (T=1: 114 MiB of the 2.4 GiB stack) while the baseline loops + over all 128 - which is why the baseline loses most in that column. + +### Roofline + +Measured on the same GPU rather than read from the spec sheet: cuBLAS bf16 +8192^3 peaks at **513.6 TFLOP/s**, a 2 GiB device copy sustains **3.51 TB/s**, so +the ridge point is **146 FLOP/byte**. + +At T=4096 (24 576 packed rows, 256 expert row tiles) every grouped GEMM has an +arithmetic intensity of 159-179 FLOP/byte - only 1.1-1.2x the ridge - so the +roofline for each stage is its *compute* time, about 0.48 ms: + +| Stage | GFLOP | GB | AI | roofline | v1 ms | of roofline | v2 ms | of roofline | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| up | 245.2 | 1.37 | 179 | 0.48 ms | 5.69 | 8.4% | 3.25 | 14.8% | +| down | 245.2 | 1.54 | 159 | 0.48 ms | 5.23 | 9.2% | 3.20 | 15.0% | +| dZ | 245.2 | 1.46 | 168 | 0.48 ms | 6.39 | 7.5% | 3.48 | 13.8% | +| dPacked | 245.2 | 1.41 | 174 | 0.48 ms | 6.26 | 7.7% | 3.44 | 13.9% | +| dW_up | 245.2 | 1.41 | 174 | 0.48 ms | 6.16 | 7.8% | 4.71 | 10.2% | +| dW_down | 245.2 | 1.37 | 179 | 0.48 ms | 6.01 | 8.0% | 4.60 | 10.4% | +| whole forward+backward | 1962 | 8.47 | 232 | 3.82 ms | 47.6 | 8.0% | 29.5 | 13.0% | + +The per-stage columns are one routed launch at T=4096 from the same +`torch.profiler` pass that produces the wall time in the last row; the headline +tables above are CUDA-graph replays over 20 samples, which is why their whole-row +medians are 1.1x higher. Byte counts assume one weight read per expert per stage +(a 9.98 MB expert stack fits the 50 MB L2) plus the activations that stage +writes. The Triton provider lands at 14.6% of the same roofline (26.2 ms). + +So the operator is **not bandwidth-bound**: at these sizes it is marginally +compute-bound, and neither provider reaches its own compute ceiling. The +[schedule sweep](#schedule-sweep) below is what ruled out the v1 path's schedule; +two structural causes remained, and both are addressed by the v2 path: + +1. **The instruction path, read off the SASS.** The v1 kernel disassembles to + `HMMA` (mma.sync) + `LDSM` (ldmatrix) with a 1552-byte local stack for the leaf + tree at 66-82 registers per thread; wgmma consumes A and B straight from + shared memory through descriptors, so v2 drops the whole ldmatrix stage + (`HGMMA 6`, `LDSM 0` per tile). +2. **No swizzle.** v1's tensor maps are pinned to `CU_TENSOR_MAP_SWIZZLE_NONE` + because the `ldmatrix` addressing is plain row-major, so a `[BK,BM]` or + `[BK,BN]` tile strides 256/128 bytes per row and every `ldmatrix` core hits a + bank-conflict set; v2 pins `SWIZZLE_128B`, which forces the 64-wide, two-leaf + K tile and a re-derived descriptor for all four operand layouts. + +Both are instruction-path decisions, not arithmetic ones: the grouped kernels +assert that the wgmma path reproduces the mma.sync path bit for bit +(`test_instruction_paths_agree_bitwise`), and the `det_gemm` parity cases still +pin the shared arithmetic. + +At decode sizes the same measurement says the opposite - T=1 moves 120 MB and +does 0.16 GFLOP, so its roofline is ~0.03 ms against 0.34 ms measured (9% for v2, +v1 was 4%): that column is latency- and padding-bound, which is why the grid +bound and the fused combine helped there and the GEMM schedule did not. The v2 +schedule helps it a different way - `BM=64` gives 6 CTAs instead of 3 on a +six-route payload, and the 2.17x forward gain at T=1 is mostly that. + +### Where the native time goes + +`torch.profiler` at T=4096, one forward+backward iteration, per routed launch +(the shared expert's six launches are the same six modes at `T` rows instead of +`6T`, and total 10.3 ms in v1 and 6.0 ms in v2): + +| Kernel | launches | v1 ms / launch | v2 ms / launch | v1/v2 | +| --- | --- | --- | --- | --- | +| `gemm` | 2 | 5.69 | 3.25 | 1.75x | +| `gemm` | 2 | 5.23 | 3.20 | 1.63x | +| `gemm` (dZ) | 2 | 6.39 | 3.48 | 1.83x | +| `gemm` (dPacked) | 2 | 6.26 | 3.44 | 1.82x | +| `gemm` (dW_up) | 2 | 6.16 | 4.71 | 1.31x | +| `gemm` (dW_down) | 2 | 6.01 | 4.60 | 1.31x | +| `dweights_leaf` | 1 | 0.47 | 0.44 | - | +| `dy` | 1 | 0.20 | 0.20 | - | +| `combine` | 1 | 0.10 | 0.10 | - | +| `dweights_merge` | 1 | 0.03 | 0.03 | - | +| total in kernels | | **47.2** | **29.7** | 1.59x | + +The weight gradient is now the slowest stage and the one that gains least: its +`K` is the expert segment length, so it pays the tail padding above and runs +`REG 170` (two CTAs per SM) against 135 for the other modes. Effective throughput +at T=4096, counting 479 MFLOP per token: v1 41 TFLOP/s, v2 66.5 TFLOP/s, Triton +75 TFLOP/s. + +### Why v2 is the contract's ceiling + +The leaf epilogue is SIMT work the contract mandates: per output element, one +BF16 rounding of the 32-wide leaf and one BF16 tree merge, i.e. one packed +instruction per two elements for each of the two operations. At T=4096 the +forward+backward performs 1962 GFLOP, which is 30.7e9 element-leaves (64 FLOP +each), so one instruction per element-leaf per thread needs 0.96e12 +warp-instructions/s. This GPU issues 132 SM x 4 schedulers x 1.785 GHz = 0.94e12, +and the measured 29.5 ms corresponds to 1.04e12 element-leaves/s. **The kernel is +at the SIMT instruction-issue ceiling** for the epilogue the contract demands, +which is the same wall that keeps `det_gemm` at 14% of peak. + +That also explains the residual gap to Triton (26.3 ms, 1.2x) and the larger gap to +the production grouped GEMM (3.59 ms against 9.85 ms at T=4096, 2.7x in the +forward): both keep an FP32 accumulator and cast once per output element instead of +once per 32-wide leaf, so they do `K/32` fewer SIMT passes - they are faster +precisely because they compute a different (non-contract) reduction. That measured +2.7x is the contract's price, and it is the number a fixed performance ratio +against native would have to be set against. Going further under this contract +means a wider leaf or a single cast, which is an arithmetic decision, not a +schedule one. + +Nsight Compute cannot be used here (`ERR_NVGPUCTRPERM`: an unprivileged container +may not open the GPU performance counters), so this section is built from +`torch.profiler` kernel attribution, `cuobjdump -res-usage` and the parameter +sweep below - all reproducible without elevated permissions. + +### Schedule sweep + +Every knob below leaves the arithmetic untouched - the leaf stays 32-wide and the +tree is unchanged - so each row is a one-constant rebuild, measured at T=4096, +forward+backward, median of 15 wall-clock iterations: + +| BM | BN | K per TMA tile | stages | wall ms | TFLOP/s | of 3.82 ms roofline | +| --- | --- | --- | --- | --- | --- | --- | +| 128 | 64 | 32 (1 leaf) | 2 | **49.04** | 40.0 | 7.8% | +| 128 | 64 | 32 | 3 | 49.00 | 40.0 | 7.8% | +| 128 | 64 | 32 | 4 | 48.77 | 40.2 | 7.8% | +| 128 | 64 | 64 (2 leaves) | 3 | 54.74 | 35.8 | 7.0% | +| 128 | 64 | 128 (4 leaves) | 2 | 57.87 | 33.9 | 6.6% | +| 128 | 64 | 128 (4 leaves) | 3 | 68.01 | 28.8 | 5.6% | +| 128 | 128 | 32 | 2 | 49.40 | 39.7 | 7.7% | +| 64 | 64 | 32 | 2 | 58.39 | 33.6 | 6.5% | +| 256 | 64 | 32 | 2, 8 warps | 49.86 | 39.4 | 7.7% | +| 128 | 64 | 32 | 2, 8 warps | 62.00 | 31.6 | 6.2% | + +* **Barriers were not the bottleneck.** One TMA tile feeding four leaves cuts the + barrier count 4x and *loses* 15%; pipeline depth 2 -> 4 changes nothing. The + first-principles "168 barriers per tile" estimate was wrong: the barrier waits + overlap with the other CTA resident on the SM. +* Wider K tiles lose because the `ldmatrix` bank-conflict degree follows the + shared-memory row stride - 64 B gives 8-way, 128 B and 256 B give 16-way. Only + TMA swizzle removes that, and swizzle caps the inner box at 128 B. +* BN=128 halves the CTA count and doubles A-fragment reuse, but it also doubles + the register accumulator; the two effects cancel. +* A second provider-level contrast, from `cuobjdump -sass`: native is + `HMMA`+`LDSM` at `REG:81 STACK:1552`, Triton is `HGMMA.64` at `REG:210 STACK:0`. + The wgmma path trades registers and occupancy for the whole ldmatrix pipeline. +* Shipped configuration: `REG:81 STACK:1552` per thread with 24.6 KB of dynamic + shared memory. The 1552-byte stack is the leaf tree itself. With the 8-register + allocation granularity that is 88 registers per thread, so the occupancy + formula allows 5 CTAs per SM by registers against 9 by shared memory: registers + bind, and the kernel runs at 20 of the 64 warp slots (31%). +* That register ceiling explains the whole table. Warp count per SM is + `65536 / (registers * 32)`, independent of the CTA shape, so any change that + enlarges the accumulator (BN=128, BM=256) loses the occupancy it gains, and any + change that shrinks the tile (BM=64, 8 warps at BM=128) pays more per-CTA + prologue and more B-fragment traffic for the same warps. + +No knob in this space moves the arithmetic, so the sweep found v1's optimum - and +that optimum is 8% of the roofline. Everything the sweep could reach is exhausted: +the remaining gap needed the two structural changes above (wgmma and swizzle), and +`RL_KERNEL_NEMOTRON_EXPERTS_BACKEND=sm90_v2` is exactly those two. They do not +break `test_matches_det_gemm_bitwise`: the wgmma path reproduces v1's reduction +bit for bit, which the parity test in +`tests/backends/cuda/test_nemotron_experts_cuda.py` asserts. + +The per-row FP64 report (`--qualify-max-tokens 128`, both providers) gives 8.7e-3 +to 1.3e-2 relative L2 for the forward and the worst gradient. One T=1 sample +reaches 2.6e-2 because a single route's dot is compared against a near-zero FP64 +sum; the unit suite, which qualifies at 17 tokens, asserts the same bounds. + +## Verification record + +Build and measurements on one NVIDIA H100 NVL (SM90, 95 GiB), CUDA 13.0, +PyTorch 2.11.0+cu130, `KERNEL_ALIGN_FORCE_SM90=1 KERNEL_ALIGN_DET_GEMM_SM90=1`: + +* `pytest tests/backends/cuda/test_nemotron_experts_cuda.py` - 28 passed, no + skips: both providers, the FP64 qualification, bitwise batch and relocation + invariance, graph replay, autocast, provider selection and rejection, the four + `det_gemm` bitwise parity cases, and the three `mma.sync`-vs-`wgmma` bitwise + parity cases (core geometry at 17 and 129 tokens, Nemotron geometry at 64); +* `pytest tests/ops/gemm/test_det_gemm.py + tests/backends/cuda/test_sm90_linear_wrappers.py + tests/backends/cuda/test_nemotron_router_cuda.py + tests/runtime/test_kernel_registry.py + tests/reference/test_nemotron_experts_reference.py + tests/reference/test_nemotron_router_reference.py + tests/backends/rocm/test_nemotron_experts_rocm.py` - 150 passed, 54 skipped + (49 of the skips are `det_gemm`'s "Triton is unavailable" and ROCm `gfx942` + rows in this build, 5 are the ROCm provider suite without a HIP device). No file + outside this operator's own sources changed, so those suites report what they + reported before; +* `torch.profiler` at T=1024 and T=4096 for both paths; the per-kernel table and + the 1.59x kernel-time ratio above come from that pass; +* `cuobjdump -res-usage` / `-sass` on the built extension for the numbers in + [Instruction paths](#instruction-paths) and + [Where the native time goes](#where-the-native-time-goes); +* the benchmark above ran four times (native v2, native v1, native v2 + Triton at + 1-8192, and the same pair at 32768); every row carries its provider id, weight + coverage, `n_e` distribution and FP64 report. The baselines of the v1 and v2 runs + agree to 0.1%; the `--reverse` order agrees to a median 0.4%; +* `black --check --line-length 100` is clean on the three python files touched and + `mypy --ignore-missing-imports` is clean on the wrapper; `mkdocs build --strict` + succeeds. + +Re-validation after the review fixes (`H100 80GB HBM3`, CUDA 13.0, PyTorch +2.12.0+cu130, same build flags): operator + reference suites 45 passed; the +`det_gemm` / router / wrapper / registry / ROCm neighbours 135 passed, 54 skipped; +`black --check`, `mypy --ignore-missing-imports` and `mkdocs --strict` clean. The +instruction-path ratio does not move on that card (T=4096 forward+backward: wgmma +27.5 ms against mma.sync 43.5 ms, 1.58x against the 1.57-1.61x above), so the +wider merge stack (`TREE_DEPTH` 12 -> 14) costs nothing measurable. The absolute +latencies in the tables remain the H100 NVL figures. + +The ROCm column has no hardware record: the same Triton source runs on CUDA, but +a gfx9xx host must re-qualify it before those numbers are quoted. The same applies +to the two native instruction paths - both are SM90 code paths, but only SM90 was +measured. diff --git a/rl_engine/_C.pyi b/rl_engine/_C.pyi index fb3c4379c..10c697c02 100644 --- a/rl_engine/_C.pyi +++ b/rl_engine/_C.pyi @@ -248,6 +248,53 @@ def det_gemm_fwd_rhs_transposed( def det_gemm_da(dc: torch.Tensor, b: torch.Tensor) -> torch.Tensor: ... def det_gemm_db(a: torch.Tensor, dc: torch.Tensor) -> torch.Tensor: ... def det_gemm_db_transposed(a: torch.Tensor, dc: torch.Tensor) -> torch.Tensor: ... +def nemotron_experts_compiled() -> bool: ... +def nemotron_experts_preferred_path() -> str: ... +def nemotron_experts_last_path() -> str: ... +def nemotron_experts_forward( + packed: torch.Tensor, + x: torch.Tensor | None, + weights: torch.Tensor, + permutation: torch.Tensor, + offsets: torch.Tensor, + w_up: torch.Tensor, + w_down: torch.Tensor, + w_sh1: torch.Tensor | None, + w_sh2: torch.Tensor | None, +) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor | None, + torch.Tensor | None, + torch.Tensor | None, +]: ... +def nemotron_experts_backward( + grad_out: torch.Tensor, + packed: torch.Tensor, + x: torch.Tensor | None, + weights: torch.Tensor, + permutation: torch.Tensor, + offsets: torch.Tensor, + up: torch.Tensor, + z: torch.Tensor, + down: torch.Tensor, + inverse: torch.Tensor, + w_up: torch.Tensor, + w_down: torch.Tensor, + w_sh1: torch.Tensor | None, + w_sh2: torch.Tensor | None, + shared_up: torch.Tensor | None, + shared_z: torch.Tensor | None, + shared_down: torch.Tensor | None, + need_packed: bool, + need_x: bool, + need_weights: bool, + need_weight_grad: bool, + need_shared_weight_grad: bool, +) -> list[torch.Tensor | None]: ... def silu_forward(x: torch.Tensor) -> torch.Tensor: ... def silu_backward(dy: torch.Tensor, x: torch.Tensor) -> torch.Tensor: ... def swiglu_forward(gate: torch.Tensor, up: torch.Tensor) -> torch.Tensor: ... diff --git a/rl_engine/backends/cuda/moe/nemotron_experts.py b/rl_engine/backends/cuda/moe/nemotron_experts.py new file mode 100644 index 000000000..1873b5310 --- /dev/null +++ b/rl_engine/backends/cuda/moe/nemotron_experts.py @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Native SM90 provider for the Nemotron grouped expert MLP. + +`csrc/cuda/moe/nemotron_experts.cu` owns the kernels; this module is the +registry entry point and the autograd wiring. The operator is strict: when the +extension was built without `KERNEL_ALIGN_DET_GEMM_SM90=1`, `__init__` raises so +`KernelRegistry` can fall through to the shared Triton provider instead of +silently running a different arithmetic. +""" + +from __future__ import annotations + +import torch + +from rl_engine.backends.extension import _C, _EXT_AVAILABLE + +PROVIDER_ID = "rlkernel.nemotron_experts.sm90.v1" + + +def _require_sm90(device: torch.device) -> None: + major, minor = torch.cuda.get_device_capability(device) + # Exactly SM90, not ">= 9": the build emits `sm_90a` (wgmma/TMA) code and + # architecture-specific `a` images are not forward compatible, so Blackwell + # must not be admitted by this gate. + if major != 9: + raise RuntimeError( + f"nemotron experts native kernel requires NVIDIA SM90, got sm_{major}{minor} " + f"on {device}" + ) + + +class NemotronExpertsOp: + """Registry entry point for the SM90 grouped expert operator.""" + + provider_id = PROVIDER_ID + + def __init__(self) -> None: + # A missing symbol means the extension predates this operator; a False + # compile flag means the SM90 path is absent from this build. Both are + # reported as instance failures so `get_op` can try the next candidate. + if not _EXT_AVAILABLE or not hasattr(_C, "nemotron_experts_forward"): + raise RuntimeError("nemotron experts native extension is not built") + if not _C.nemotron_experts_compiled(): + raise RuntimeError( + "nemotron experts native kernel requires KERNEL_ALIGN_DET_GEMM_SM90=1" + ) + + def __call__( + self, + packed: torch.Tensor, + x: torch.Tensor | None, + weights: torch.Tensor, + permutation: torch.Tensor, + offsets: torch.Tensor, + w_up: torch.Tensor, + w_down: torch.Tensor, + w_sh1: torch.Tensor | None = None, + w_sh2: torch.Tensor | None = None, + ) -> torch.Tensor: + # The build flag says the kernel exists, not that `packed`'s device can + # run it: `KERNEL_ALIGN_DET_GEMM_SM90=1` compiles the SM90 paths from + # the build host's architecture, while the registry lists this provider + # first for every CUDA device and caches the instance for the process. + # The check therefore follows the payload device on every call, not + # `torch.cuda.current_device()` once at construction, so a pre-Hopper + # device fails closed and a Hopper device stays usable in one process. + _require_sm90(packed.device) + return _ExpertsCombine.apply( + packed, x, weights, permutation, offsets, w_up, w_down, w_sh1, w_sh2 + ) + + +class _ExpertsCombine(torch.autograd.Function): + @staticmethod + def forward( + ctx, + packed, + x, + weights, + permutation, + offsets, + w_up, + w_down, + w_sh1, + w_sh2, + ): + out, up, z, down, inverse, shared_up, shared_z, shared_down = _C.nemotron_experts_forward( + packed, x, weights, permutation, offsets, w_up, w_down, w_sh1, w_sh2 + ) + ctx.save_for_backward( + packed, + x, + weights, + permutation, + offsets, + up, + z, + down, + inverse, + w_up, + w_down, + w_sh1, + w_sh2, + shared_up, + shared_z, + shared_down, + ) + ctx.mark_non_differentiable(permutation, offsets) + ctx.set_materialize_grads(False) + return out + + @staticmethod + def backward(ctx, grad_out): + ( + packed, + x, + weights, + permutation, + offsets, + up, + z, + down, + inverse, + w_up, + w_down, + w_sh1, + w_sh2, + shared_up, + shared_z, + shared_down, + ) = ctx.saved_tensors + needs = ctx.needs_input_grad + if grad_out is None: + return (None,) * 9 + shared = w_sh1 is not None + need_shared_weight_grad = shared and (bool(needs[7]) or bool(needs[8])) + need_any = ( + bool(needs[0]) + or (shared and bool(needs[1])) + or bool(needs[2]) + or bool(needs[5]) + or bool(needs[6]) + or need_shared_weight_grad + ) + if not need_any: + return (None,) * 9 + # Autograd does not guarantee a contiguous incoming gradient (a narrowed + # or transposed output produces a strided one); the kernel requires it. + d_packed, d_x, d_weights, d_w_up, d_w_down, d_w_sh1, d_w_sh2 = _C.nemotron_experts_backward( + grad_out.contiguous(), + packed, + x, + weights, + permutation, + offsets, + up, + z, + down, + inverse, + w_up, + w_down, + w_sh1, + w_sh2, + shared_up, + shared_z, + shared_down, + bool(needs[0]), + shared and bool(needs[1]), + bool(needs[2]), + bool(needs[5]) or bool(needs[6]), + need_shared_weight_grad, + ) + # Undefined extension results already arrive as `None`; a zero-element + # tensor is a real gradient (the zero-token payload case) and keeps its + # shape instead of collapsing to "absent". + return (d_packed, d_x, d_weights, None, None, d_w_up, d_w_down, d_w_sh1, d_w_sh2) diff --git a/rl_engine/backends/shared/triton/moe/__init__.py b/rl_engine/backends/shared/triton/moe/__init__.py new file mode 100644 index 000000000..2d440d4ce --- /dev/null +++ b/rl_engine/backends/shared/triton/moe/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Package extension point; implementations are registered explicitly.""" diff --git a/rl_engine/backends/shared/triton/moe/nemotron_experts.py b/rl_engine/backends/shared/triton/moe/nemotron_experts.py new file mode 100644 index 000000000..846572d11 --- /dev/null +++ b/rl_engine/backends/shared/triton/moe/nemotron_experts.py @@ -0,0 +1,795 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Shared Triton provider for the Nemotron grouped expert MLP (CUDA and ROCm). + +One source for both platforms: the CUDA column falls back to it when the SM90 +extension is absent, and it is the only ROCm implementation of +`moe_experts_combine`. Pinned schedule, no autotuning, no split-K, no atomics: +`BLOCK_M=128`, `BLOCK_N=64`, `BLOCK_K=32` (one FP32 leaf), 4 warps, 2 stages and +`enable_fp_fusion=False` on every launch, because a per-shape configuration +choice would itself break batch invariance. + +Arithmetic, matching `nemotron-experts-sm90-v1` where Triton can express it: + +* K is cut into 32-wide leaves; each leaf is one FP32 `tl.dot` rounded once to + BF16, the same leaf width and the same single rounding as the native kernel + (the in-leaf accumulation order is the backend's own, exactly as `mma.sync` + and a scalar MAC differ inside one BF16 product); +* leaves fold in ascending order. The native provider merges them with the + canonical midpoint tree; Triton's frontend cannot build a trace-time stack + (kernels may not call plain-Python helpers, and `tl.static_range` forbids the + list operations the stack algorithm needs), so the association differs. Folding + is still fixed, atomic-free and batch invariant, and each provider is + qualified against FP64 on its own branch; +* everything else is identical to the native provider: the relu-square leaf + rounding, the FP32 down-projection, the ascending-slot FP32 combine, the single + exit cast, and `dWeights` accumulating in FP32. + +Cross-provider bitwise equality is not claimed. +""" + +from __future__ import annotations + +import torch + +try: + import triton + import triton.language as tl + + _TRITON_AVAILABLE = True +except ImportError: # pragma: no cover - installations without Triton + _TRITON_AVAILABLE = False + +PROVIDER_ID = "rlkernel.nemotron_experts.triton.v1" + +BLOCK_M = 128 +BLOCK_N = 64 +LEAF = 32 +NUM_WARPS = 4 +NUM_STAGES = 2 +# Partial sums accumulate in FP32 and round to BF16 once per window of leaves. +GROUP_LEAVES = 8 +_ELEMENT_BLOCK = 1024 + +if _TRITON_AVAILABLE: + + @triton.jit + def _resolve_segment( + OFFSETS, tile, rows, BM: tl.constexpr, NSEG: tl.constexpr, NSEG_P2: tl.constexpr + ): + """Map a row-tile index to (expert, first row, segment end, tile count).""" + experts = tl.arange(0, NSEG_P2) + known = experts < NSEG + begin = tl.load(OFFSETS + experts, known, 0) + end = tl.load(OFFSETS + experts + 1, known, 0) + tiles = (end - begin + BM - 1) // BM + prefix = tl.cumsum(tiles, 0) + total = tl.sum(tiles, 0) + expert = tl.minimum(tl.min(tl.where(prefix > tile, experts, NSEG_P2)), NSEG - 1) + before = tl.sum(tl.where(experts < expert, tiles, 0), 0) + first = tl.sum(tl.where(experts == expert, begin, 0), 0) + last = tl.sum(tl.where(experts == expert, end, 0), 0) + return expert, first + (tile - before) * BM, last, total + + @triton.jit + def _invert_kernel(INVERSE, PERMUTATION, routes, BLOCK: tl.constexpr): + index = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + mask = index < routes + # The router permutation is a bijection, so this scatter cannot collide. + target = tl.load(PERMUTATION + index, mask, 0) + tl.store(INVERSE + target, index.to(tl.int64), mask) + + @triton.jit + def _grouped_rows_kernel( + A, + W, + C, + OFFSETS, + rows, + N, + K, + RELU2_A: tl.constexpr, + OUT_FP32: tl.constexpr, + NSEG: tl.constexpr, + NSEG_P2: tl.constexpr, + SINGLE: tl.constexpr, + BM: tl.constexpr, + BN: tl.constexpr, + BK: tl.constexpr, + GROUP: tl.constexpr, + ): + """Grouped GEMM with the weights stacked as [experts, N, K].""" + tile = tl.program_id(0) + col = tl.program_id(1) * BN + if SINGLE: + total = (rows + BM - 1) // BM + if tile >= total: + return + expert = 0 + first = tile * BM + last = rows + else: + expert, first, last, total = _resolve_segment(OFFSETS, tile, rows, BM, NSEG, NSEG_P2) + if tile >= total: + return + rm = tl.arange(0, BM) + rn = col + tl.arange(0, BN) + row_mask = first + rm < last + col_mask = rn < N + weight_base = W + (expert * N + rn)[:, None] * K + total_acc = tl.zeros((BM, BN), tl.bfloat16) + window = tl.zeros((BM, BN), tl.float32) + for leaf in tl.range(0, K // BK): + koff = leaf * BK + tl.arange(0, BK) + a = tl.load(A + (first + rm)[:, None] * K + koff[None, :], row_mask[:, None], 0.0) + b = tl.load(weight_base + koff[None, :], col_mask[:, None], 0.0) + if RELU2_A: + gate = tl.maximum(a.to(tl.float32), 0.0) + a = (gate * gate).to(tl.bfloat16) + # One BF16 rounding per 32-wide leaf, then FP32 partial sums that + # round once per window instead of once per leaf. + window += tl.dot(a, tl.trans(b)).to(tl.bfloat16).to(tl.float32) + flush = (leaf + 1) % GROUP == 0 + total_acc = tl.where( + flush, (total_acc + window.to(tl.bfloat16)).to(tl.bfloat16), total_acc + ) + window = tl.where(flush, 0.0, window) + total_acc = (total_acc + window.to(tl.bfloat16)).to(tl.bfloat16) + mask = row_mask[:, None] & col_mask[None, :] + out = C + (first + rm)[:, None] * N + rn[None, :] + if OUT_FP32: + tl.store(out, total_acc.to(tl.float32), mask) + else: + tl.store(out, total_acc, mask) + + @triton.jit + def _grouped_kn_kernel( + A, + W, + C, + AUX, + OFFSETS, + rows, + N, + K, + EPI_RELU2_GRAD: tl.constexpr, + NSEG: tl.constexpr, + NSEG_P2: tl.constexpr, + SINGLE: tl.constexpr, + BM: tl.constexpr, + BN: tl.constexpr, + BK: tl.constexpr, + GROUP: tl.constexpr, + ): + """Grouped GEMM with the weights stacked as [experts, K, N] (backward).""" + tile = tl.program_id(0) + col = tl.program_id(1) * BN + if SINGLE: + total = (rows + BM - 1) // BM + if tile >= total: + return + expert = 0 + first = tile * BM + last = rows + else: + expert, first, last, total = _resolve_segment(OFFSETS, tile, rows, BM, NSEG, NSEG_P2) + if tile >= total: + return + rm = tl.arange(0, BM) + rn = col + tl.arange(0, BN) + row_mask = first + rm < last + col_mask = rn < N + weight_base = W + (expert * K + tl.arange(0, BK))[:, None] * N + total_acc = tl.zeros((BM, BN), tl.bfloat16) + window = tl.zeros((BM, BN), tl.float32) + for leaf in tl.range(0, K // BK): + koff = leaf * BK + a = tl.load( + A + (first + rm)[:, None] * K + (koff + tl.arange(0, BK))[None, :], + row_mask[:, None], + 0.0, + ) + b = tl.load(weight_base + koff * N + rn[None, :], col_mask[None, :], 0.0) + window += tl.dot(a, b).to(tl.bfloat16).to(tl.float32) + flush = (leaf + 1) % GROUP == 0 + total_acc = tl.where( + flush, (total_acc + window.to(tl.bfloat16)).to(tl.bfloat16), total_acc + ) + window = tl.where(flush, 0.0, window) + total_acc = (total_acc + window.to(tl.bfloat16)).to(tl.bfloat16) + mask = row_mask[:, None] & col_mask[None, :] + value = total_acc.to(tl.float32) + if EPI_RELU2_GRAD: + up = tl.load(AUX + (first + rm)[:, None] * N + rn[None, :], mask, 0.0).to(tl.float32) + value = value * (2.0 * tl.maximum(up, 0.0)) + tl.store(C + (first + rm)[:, None] * N + rn[None, :], value.to(tl.bfloat16), mask) + + @triton.jit + def _wgrad_kernel( + A, + B, + C, + OFFSETS, + rows, + M, + N, + RELU2_B: tl.constexpr, + NSEG: tl.constexpr, + SINGLE: tl.constexpr, + BM: tl.constexpr, + BN: tl.constexpr, + BK: tl.constexpr, + GROUP: tl.constexpr, + ): + """dW[e, m, n] = sum over the expert's packed rows of A[r, m] * B[r, n].""" + expert = tl.program_id(2) + rm = tl.program_id(0) * BM + tl.arange(0, BM) + rn = tl.program_id(1) * BN + tl.arange(0, BN) + if SINGLE: + first = 0 + last = rows + else: + first = tl.load(OFFSETS + expert) + last = tl.load(OFFSETS + expert + 1) + m_mask = rm < M + n_mask = rn < N + keep = m_mask[:, None] & n_mask[None, :] + out = C + (expert * M + rm)[:, None] * N + rn[None, :] + # Every expert owns a tile, so an empty segment writes exact zeros. + total = tl.zeros((BM, BN), tl.bfloat16) + window = tl.zeros((BM, BN), tl.float32) + a_base = A + rm[None, :] + b_base = B + rn[None, :] + for leaf in tl.range(0, tl.cdiv(last - first, BK)): + rows_at = first + leaf * BK + tl.arange(0, BK) + row_mask = rows_at < last + a = tl.load(a_base + rows_at[:, None] * M, row_mask[:, None] & m_mask[None, :], 0.0) + b = tl.load(b_base + rows_at[:, None] * N, row_mask[:, None] & n_mask[None, :], 0.0) + if RELU2_B: + gate = tl.maximum(b.to(tl.float32), 0.0) + b = (gate * gate).to(tl.bfloat16) + window += tl.dot(tl.trans(a), b).to(tl.bfloat16).to(tl.float32) + flush = (leaf + 1) % GROUP == 0 + total = tl.where(flush, (total + window.to(tl.bfloat16)).to(tl.bfloat16), total) + window = tl.where(flush, 0.0, window) + total = (total + window.to(tl.bfloat16)).to(tl.bfloat16) + tl.store(out, total, keep) + + @triton.jit + def _combine_kernel( + DOWN, INVERSE, WEIGHTS, SHARED, OUT, width, HAS_SHARED: tl.constexpr, BLOCK: tl.constexpr + ): + """Combine, shared add and the operator's only exit cast in one pass.""" + token = tl.program_id(0) + cols = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) + mask = cols < width + base = token * 6 + offset = token * width + cols + total = tl.zeros((BLOCK,), tl.float32) + for slot in tl.static_range(6): + row = tl.load(INVERSE + base + slot) + weight = tl.load(WEIGHTS + base + slot) + total += weight * tl.load(DOWN + row * width + cols, mask, 0.0) + if HAS_SHARED: + total += tl.load(SHARED + offset, mask, 0.0) + tl.store(OUT + offset, total.to(tl.bfloat16), mask) + + @triton.jit + def _dy_kernel(DY, GRAD, PERMUTATION, WEIGHTS, width, BLOCK: tl.constexpr): + row = tl.program_id(0) + cols = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) + mask = cols < width + original = tl.load(PERMUTATION + row) + weight = tl.load(WEIGHTS + original) + value = tl.load(GRAD + (original // 6) * width + cols, mask, 0.0).to(tl.float32) + tl.store(DY + row * width + cols, (weight * value).to(tl.bfloat16), mask) + + @triton.jit + def _dweights_kernel(DOWN, GRAD, INVERSE, OUT, width, BK: tl.constexpr): + """FP32 route dot product; one program per route, slots come from INV.""" + route = tl.program_id(0) + row = tl.load(INVERSE + route) + token = route // 6 + cols = tl.arange(0, BK) + down_base = DOWN + row * width + grad_base = GRAD + token * width + total = tl.zeros((BK,), tl.float32) + for leaf in tl.range(0, width // BK): + offset = leaf * BK + cols + total += tl.load(down_base + offset) * tl.load(grad_base + offset).to(tl.float32) + tl.store(OUT + route, tl.sum(total, 0)) + + +def _next_power_of_two(value: int) -> int: + return 1 << max(0, (value - 1).bit_length()) + + +class TritonNemotronExpertsOp: + """Registry entry point for the shared Triton grouped expert operator.""" + + provider_id = PROVIDER_ID + + def __init__(self) -> None: + if not _TRITON_AVAILABLE: + raise RuntimeError("triton is not installed; the shared experts provider needs it") + if torch.version.hip is None and not torch.cuda.is_available(): + raise RuntimeError("the shared experts provider needs a CUDA or ROCm device") + + def __call__( + self, + packed: torch.Tensor, + x: torch.Tensor | None, + weights: torch.Tensor, + permutation: torch.Tensor, + offsets: torch.Tensor, + w_up: torch.Tensor, + w_down: torch.Tensor, + w_sh1: torch.Tensor | None = None, + w_sh2: torch.Tensor | None = None, + ) -> torch.Tensor: + return _ExpertsCombine.apply( + packed, x, weights, permutation, offsets, w_up, w_down, w_sh1, w_sh2 + ) + + +def _rows_gemm( + source, + weight, + out, + aux, + offsets, + rows, + reduction, + width, + experts, + *, + transpose_weights: bool, + relu2_source: bool = False, + out_fp32: bool = False, + epi_relu2_grad: bool = False, +): + """One grouped GEMM; `offsets=None` means the single segment [0, rows).""" + single = offsets is None + kernel = _grouped_kn_kernel if transpose_weights else _grouped_rows_kernel + extra = ( + {"EPI_RELU2_GRAD": epi_relu2_grad} + if transpose_weights + else {"RELU2_A": relu2_source, "OUT_FP32": out_fp32} + ) + grid = ( + triton.cdiv(rows, BLOCK_M) + 1 if single else triton.cdiv(rows, BLOCK_M) + experts, + triton.cdiv(width, BLOCK_N), + ) + # The backward kernels take the relu-square source for the derivative + # epilogue; the forward kernels have no such operand. + operands = ( + (source, weight, out, aux if aux is not None else out, offsets, rows) + if transpose_weights + else (source, weight, out, offsets, rows) + ) + kernel[grid]( + *operands, + N=width, + K=reduction, + NSEG=1 if single else experts, + NSEG_P2=1 if single else _next_power_of_two(experts), + SINGLE=single, + BM=BLOCK_M, + BN=BLOCK_N, + BK=LEAF, + GROUP=GROUP_LEAVES, + **extra, + num_warps=NUM_WARPS, + num_stages=NUM_STAGES, + enable_fp_fusion=False, + ) + + +def _check_leaf_alignment(**widths): + """Refuse a reduction width the leaf loop would truncate. + + `_rows_gemm` and `_dweights_kernel` step the reduction in whole `LEAF`-wide + leaves without a tail mask (the masked `_wgrad_kernel` covers the one place + where the reduction length is arbitrary). The operator's validator already + requires 64-aligned widths, so this only rejects a hand-built provider call - + but silently dropping the tail leaf would be a wrong answer, not a slow one. + """ + for name, value in widths.items(): + if value % LEAF: + raise ValueError( + f"nemotron experts (triton): {name}={value} is not a multiple of the " + f"{LEAF}-wide leaf" + ) + + +def _forward(packed, x, weights, permutation, offsets, w_up, w_down, w_sh1, w_sh2): + rows, width = packed.shape + tokens = rows // 6 + experts, intermediate = w_up.shape[0], w_up.shape[1] + device = packed.device + dtype = packed.dtype + _check_leaf_alignment(width=width, intermediate=intermediate) + if w_sh1 is not None: + _check_leaf_alignment(shared_width=w_sh1.shape[0]) + up = torch.empty((rows, intermediate), device=device, dtype=dtype) + down = torch.empty((rows, width), device=device, dtype=torch.float32) + inverse = torch.empty((rows,), device=device, dtype=torch.int64) + if tokens == 0: + return torch.empty((0, width), device=device, dtype=dtype), up, down, inverse, None, None + _invert_kernel[(triton.cdiv(rows, 256),)](inverse, permutation, rows, 256) + _rows_gemm( + packed, + w_up, + up, + None, + offsets, + rows, + width, + intermediate, + experts, + transpose_weights=False, + ) + _rows_gemm( + up, + w_down, + down, + None, + offsets, + rows, + intermediate, + width, + experts, + transpose_weights=False, + relu2_source=True, + out_fp32=True, + ) + shared_up = shared_down = None + if w_sh1 is not None: + shared_width = w_sh1.shape[0] + shared_up = torch.empty((tokens, shared_width), device=device, dtype=dtype) + shared_down = torch.empty((tokens, width), device=device, dtype=torch.float32) + _rows_gemm( + x, + w_sh1, + shared_up, + None, + None, + tokens, + width, + shared_width, + 1, + transpose_weights=False, + ) + _rows_gemm( + shared_up, + w_sh2, + shared_down, + None, + None, + tokens, + shared_width, + width, + 1, + transpose_weights=False, + relu2_source=True, + out_fp32=True, + ) + out = torch.empty((tokens, width), device=device, dtype=dtype) + # Combine, shared add and the single exit cast in one pass over the tokens. + _combine_kernel[(tokens, triton.cdiv(width, _ELEMENT_BLOCK))]( + down, + inverse, + weights, + shared_down if shared_down is not None else out, + out, + width, + shared_down is not None, + _ELEMENT_BLOCK, + num_warps=4, + enable_fp_fusion=False, + ) + return out, up, down, inverse, shared_up, shared_down + + +def _backward( + grad_out, + packed, + x, + weights, + permutation, + offsets, + up, + down, + inverse, + w_up, + w_down, + w_sh1, + w_sh2, + shared_up, + need_packed, + need_x, + need_weights, + need_weight_grad, + need_shared_weight_grad, +): + rows, width = packed.shape + tokens = rows // 6 + experts, intermediate = w_up.shape[0], w_up.shape[1] + device = packed.device + dtype = packed.dtype + _check_leaf_alignment(width=width, intermediate=intermediate) + if w_sh1 is not None: + _check_leaf_alignment(shared_width=w_sh1.shape[0]) + if tokens == 0: + # An expert-parallel rank may hold no tokens. Every returned gradient + # still has to be shaped like its input: an empty-row payload gets an + # empty payload gradient, and a weight gradient is exactly zero (not + # absent), so an optimizer step and a DDP all-reduce stay well defined. + shared_grad = need_shared_weight_grad and w_sh1 is not None and w_sh2 is not None + return [ + torch.zeros((rows, width), device=device, dtype=dtype) if need_packed else None, + torch.zeros((tokens, width), device=device, dtype=dtype) if need_x else None, + ( + torch.zeros((tokens, 6), device=device, dtype=torch.float32) + if need_weights + else None + ), + torch.zeros_like(w_up) if need_weight_grad else None, + torch.zeros_like(w_down) if need_weight_grad else None, + torch.zeros_like(w_sh1) if shared_grad else None, + torch.zeros_like(w_sh2) if shared_grad else None, + ] + dy = torch.empty((rows, width), device=device, dtype=dtype) + _dy_kernel[(rows, triton.cdiv(width, _ELEMENT_BLOCK))]( + dy, + grad_out, + permutation, + weights, + width, + _ELEMENT_BLOCK, + num_warps=4, + enable_fp_fusion=False, + ) + d_weights = None + if need_weights: + d_weights = torch.empty((tokens, 6), device=device, dtype=torch.float32) + _dweights_kernel[(rows,)]( + down, grad_out, inverse, d_weights, width, LEAF, num_warps=1, enable_fp_fusion=False + ) + du = None + if need_packed or need_weight_grad: + du = torch.empty((rows, intermediate), device=device, dtype=dtype) + _rows_gemm( + dy, + w_down, + du, + up, + offsets, + rows, + width, + intermediate, + experts, + transpose_weights=True, + epi_relu2_grad=True, + ) + d_packed = None + if need_packed: + d_packed = torch.empty((rows, width), device=device, dtype=dtype) + _rows_gemm( + du, + w_up, + d_packed, + None, + offsets, + rows, + intermediate, + width, + experts, + transpose_weights=True, + ) + d_w_up = d_w_down = None + if need_weight_grad: + d_w_up = torch.empty_like(w_up) + d_w_down = torch.empty_like(w_down) + _wgrad_kernel[(triton.cdiv(intermediate, BLOCK_M), triton.cdiv(width, BLOCK_N), experts)]( + du, + packed, + d_w_up, + offsets, + rows, + intermediate, + width, + False, + experts, + False, + BLOCK_M, + BLOCK_N, + LEAF, + GROUP_LEAVES, + num_warps=NUM_WARPS, + num_stages=NUM_STAGES, + enable_fp_fusion=False, + ) + _wgrad_kernel[(triton.cdiv(width, BLOCK_M), triton.cdiv(intermediate, BLOCK_N), experts)]( + dy, + up, + d_w_down, + offsets, + rows, + width, + intermediate, + True, + experts, + False, + BLOCK_M, + BLOCK_N, + LEAF, + GROUP_LEAVES, + num_warps=NUM_WARPS, + num_stages=NUM_STAGES, + enable_fp_fusion=False, + ) + d_x = d_w_sh1 = d_w_sh2 = None + if w_sh1 is not None: + shared_width = w_sh1.shape[0] + d_shared_up = None + if need_x or need_shared_weight_grad: + d_shared_up = torch.empty((tokens, shared_width), device=device, dtype=dtype) + _rows_gemm( + grad_out, + w_sh2, + d_shared_up, + shared_up, + None, + tokens, + width, + shared_width, + 1, + transpose_weights=True, + epi_relu2_grad=True, + ) + if need_x: + d_x = torch.empty((tokens, width), device=device, dtype=dtype) + _rows_gemm( + d_shared_up, + w_sh1, + d_x, + None, + None, + tokens, + shared_width, + width, + 1, + transpose_weights=True, + ) + if need_shared_weight_grad: + # Same shape as the inputs: `_wgrad_kernel` writes `[M, N]` from the + # group offset, which is byte-identical for a 2-D allocation. + d_w_sh1 = torch.empty((shared_width, width), device=device, dtype=dtype) + d_w_sh2 = torch.empty((width, shared_width), device=device, dtype=dtype) + _wgrad_kernel[(triton.cdiv(shared_width, BLOCK_M), triton.cdiv(width, BLOCK_N), 1)]( + d_shared_up, + x, + d_w_sh1, + None, + tokens, + shared_width, + width, + False, + 1, + True, + BLOCK_M, + BLOCK_N, + LEAF, + GROUP_LEAVES, + num_warps=NUM_WARPS, + num_stages=NUM_STAGES, + enable_fp_fusion=False, + ) + _wgrad_kernel[(triton.cdiv(width, BLOCK_M), triton.cdiv(shared_width, BLOCK_N), 1)]( + grad_out, + shared_up, + d_w_sh2, + None, + tokens, + width, + shared_width, + True, + 1, + True, + BLOCK_M, + BLOCK_N, + LEAF, + GROUP_LEAVES, + num_warps=NUM_WARPS, + num_stages=NUM_STAGES, + enable_fp_fusion=False, + ) + return [d_packed, d_x, d_weights, d_w_up, d_w_down, d_w_sh1, d_w_sh2] + + +class _ExpertsCombine(torch.autograd.Function): + @staticmethod + def forward(ctx, packed, x, weights, permutation, offsets, w_up, w_down, w_sh1, w_sh2): + out, up, down, inverse, shared_up, shared_down = _forward( + packed, x, weights, permutation, offsets, w_up, w_down, w_sh1, w_sh2 + ) + ctx.save_for_backward( + packed, + x, + weights, + permutation, + offsets, + up, + down, + inverse, + w_up, + w_down, + w_sh1, + w_sh2, + shared_up, + shared_down, + ) + ctx.mark_non_differentiable(permutation, offsets) + ctx.set_materialize_grads(False) + return out + + @staticmethod + def backward(ctx, grad_out): + ( + packed, + x, + weights, + permutation, + offsets, + up, + down, + inverse, + w_up, + w_down, + w_sh1, + w_sh2, + shared_up, + shared_down, + ) = ctx.saved_tensors + needs = ctx.needs_input_grad + if grad_out is None: + return (None,) * 9 + shared = w_sh1 is not None + need_shared_weight_grad = shared and (bool(needs[7]) or bool(needs[8])) + need_any = ( + bool(needs[0]) + or (shared and bool(needs[1])) + or bool(needs[2]) + or bool(needs[5]) + or bool(needs[6]) + or need_shared_weight_grad + ) + if not need_any: + return (None,) * 9 + gradients = _backward( + grad_out.contiguous(), + packed, + x, + weights, + permutation, + offsets, + up, + down, + inverse, + w_up, + w_down, + w_sh1, + w_sh2, + shared_up, + bool(needs[0]), + shared and bool(needs[1]), + bool(needs[2]), + bool(needs[5]) or bool(needs[6]), + need_shared_weight_grad, + ) + d_packed, d_x, d_weights, d_w_up, d_w_down, d_w_sh1, d_w_sh2 = gradients + # `_backward` returns `None` for a gradient it was not asked for and a + # real tensor otherwise; a zero-token payload's empty row gradient is + # still shaped like its input, so it must not be coerced to "absent". + return (d_packed, d_x, d_weights, None, None, d_w_up, d_w_down, d_w_sh1, d_w_sh2) diff --git a/rl_engine/ops/moe/nemotron_experts.py b/rl_engine/ops/moe/nemotron_experts.py new file mode 100644 index 000000000..879b4bd56 --- /dev/null +++ b/rl_engine/ops/moe/nemotron_experts.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Hardware-independent entry for the Nemotron grouped expert MLP. + +The `moe_experts_combine` row of RFC #434: grouped up-projection GEMM with a +relu-square epilogue, grouped down-projection, a fixed-order weighted combine +over the six route slots and the optional shared expert, with one exit cast. + +This module owns the only copy of the argument validation and selects a +provider through the kernel registry, so the CUDA and ROCm columns implement one +versioned contract. The hot path never synchronizes: `permutation` and +`offsets` values are the router's responsibility (a bijection and exclusive +prefix sums), and only shape/dtype/device metadata is checked here. +""" + +from __future__ import annotations + +import threading + +import torch + +from rl_engine.runtime.policy import rl_kernel_mode, route_report_enabled +from rl_engine.runtime.registry import kernel_registry + +CONTRACT_VERSION = "nemotron-experts-sm90-v1" +OP_TYPE = "moe_experts_combine" +# Bound every 32-bit index product: six packed rows per token, 2688 columns. +MAX_TOKENS = 65536 +# Wide enough for the vectorized stages and for the 64-wide GEMM tiles. +_WIDTH_ALIGNMENT = 64 +_VECTOR_ALIGNMENT = 8 + +_PROVIDER_REPORTED = False +_PROVIDER_REPORT_LOCK = threading.Lock() + + +def _require(name, tensor, dtype, *, dims=None, device=None, contiguous=True): + if not isinstance(tensor, torch.Tensor): + raise ValueError(f"{name} must be a torch.Tensor") + if tensor.dtype != dtype: + raise ValueError(f"{name} must be {str(dtype).replace('torch.', '')}") + if tensor.device.type != "cuda": + raise ValueError(f"{name} must be a CUDA tensor") + if device is not None and tensor.device != device: + raise ValueError(f"{name} must live on the payload device") + if contiguous and not tensor.is_contiguous(): + raise ValueError(f"{name} must be contiguous") + if dims is not None and tensor.dim() != dims: + raise ValueError(f"{name} must be rank {dims}") + return tensor + + +def _validate( + packed, + x, + weights, + permutation, + offsets, + w_up, + w_down, + w_sh1, + w_sh2, +): + _require("packed", packed, torch.bfloat16, dims=2) + device = packed.device + tokens, width = packed.shape + if tokens % 6: + raise ValueError("packed must hold six rows per token") + tokens //= 6 + if tokens > MAX_TOKENS: + raise ValueError(f"at most {MAX_TOKENS} tokens supported by this operator") + if width % _WIDTH_ALIGNMENT or width % _VECTOR_ALIGNMENT: + raise ValueError(f"hidden width must be {_WIDTH_ALIGNMENT} aligned") + _require("weights", weights, torch.float32, dims=2, device=device) + if weights.shape != (tokens, 6): + raise ValueError("weights must be [tokens, 6] float32") + _require("permutation", permutation, torch.int64, dims=1, device=device) + if permutation.numel() != tokens * 6: + raise ValueError("permutation must cover every packed row") + _require("offsets", offsets, torch.int64, dims=1, device=device) + _require("w_up", w_up, torch.bfloat16, dims=3, device=device) + _require("w_down", w_down, torch.bfloat16, dims=3, device=device) + experts, intermediate = w_up.shape[0], w_up.shape[1] + if experts < 1: + raise ValueError("at least one expert is required") + if w_up.shape[2] != width: + raise ValueError("w_up hidden width must match packed") + if w_down.shape != (experts, width, intermediate): + raise ValueError("w_down must be [experts, hidden, intermediate]") + if intermediate % _WIDTH_ALIGNMENT: + raise ValueError(f"intermediate width must be {_WIDTH_ALIGNMENT} aligned") + if offsets.numel() != experts + 1: + raise ValueError("offsets must hold experts+1 exclusive prefix sums") + if (w_sh1 is None) != (w_sh2 is None): + raise ValueError("shared expert weights must be supplied as a pair") + if w_sh1 is not None: + _require("w_sh1", w_sh1, torch.bfloat16, dims=2, device=device) + _require("w_sh2", w_sh2, torch.bfloat16, dims=2, device=device) + if x is None: + raise ValueError("the shared expert requires the token payload x") + if w_sh1.shape[1] != width or w_sh2.shape[0] != width: + raise ValueError("shared expert hidden widths must agree") + if w_sh1.shape[0] != w_sh2.shape[1]: + raise ValueError("shared expert intermediate widths must agree") + if w_sh1.shape[0] % _WIDTH_ALIGNMENT: + raise ValueError(f"shared intermediate width must be {_WIDTH_ALIGNMENT} aligned") + if x is not None: + _require("x", x, torch.bfloat16, dims=2, device=device) + if tuple(x.shape) != (tokens, width): + raise ValueError("x must be [tokens, hidden]") + + +def _report_provider_once(op) -> None: + global _PROVIDER_REPORTED + if torch._dynamo.is_compiling(): + return + if not route_report_enabled(): + return + with _PROVIDER_REPORT_LOCK: + if _PROVIDER_REPORTED: + return + _PROVIDER_REPORTED = True + print( + f"[RL-Kernel][route] mode={rl_kernel_mode().value} module=moe_experts " + f"provider={nemotron_experts_provider_id(op)}", + flush=True, + ) + + +def nemotron_experts_provider_id(op=None) -> str: + """Return the provider id that would run `moe_experts_combine` on this device. + + Pass an already selected operator to avoid a second registry lookup. + """ + if op is None: + op = kernel_registry.get_op(OP_TYPE) + return str(getattr(op, "provider_id", type(op).__name__)) + + +def nemotron_experts( + packed: torch.Tensor, + x: torch.Tensor | None, + weights: torch.Tensor, + permutation: torch.Tensor, + offsets: torch.Tensor, + w_up: torch.Tensor, + w_down: torch.Tensor, + w_sh1: torch.Tensor | None = None, + w_sh2: torch.Tensor | None = None, +) -> torch.Tensor: + """Run the grouped expert MLP, combine and shared expert add. + + `packed` is the router payload `[6*tokens, hidden]` and stays differentiable: + its gradient is the route-weighted, down-projected expert gradient. `x` is + the original token payload used only by the shared expert, so it may be None + when `w_sh1`/`w_sh2` are omitted. Every input must be contiguous, on one + CUDA device, and `permutation`/`offsets` must be the router's bijection and + exclusive prefix sums; their values are not re-validated on the hot path. + """ + _validate(packed, x, weights, permutation, offsets, w_up, w_down, w_sh1, w_sh2) + op = kernel_registry.get_op(OP_TYPE, device=packed.device) + _report_provider_once(op) + return op(packed, x, weights, permutation, offsets, w_up, w_down, w_sh1, w_sh2) diff --git a/rl_engine/reference/moe/nemotron_experts.py b/rl_engine/reference/moe/nemotron_experts.py new file mode 100644 index 000000000..5873b0a9d --- /dev/null +++ b/rl_engine/reference/moe/nemotron_experts.py @@ -0,0 +1,197 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Nemotron Nano grouped expert MLP + fixed-order combine specification. + +One differentiable reference for the `moe_experts_combine` row of RFC #434: +grouped up-projection GEMM with a relu-square epilogue, grouped down-projection, +a fixed-order weighted combine over the six route slots and the optional shared +expert. This module fixes the *semantics*; it does not reproduce the kernels' +mid-split BF16 arithmetic. + +FP64 inputs stay FP64 end to end, so `gradcheck` and the scalar oracle see the +idealized composition. BF16/FP32 inputs model the two cast points that dominate +the error: the up-projection result and the relu-square gate round to BF16, the +down projection and the combine accumulate in FP32, and there is exactly one +exit cast. The kernels' `det_gemm` contract additionally rounds every 32-wide +reduction leaf to BF16 before merging the leaves - in the down projection and +the backward GEMMs too - which this baseline does not reproduce, because +`torch.matmul` computes the whole reduction in FP32. It is therefore a +numerical baseline, not a bitwise contract; the kernels' arithmetic is pinned +separately against `_C.det_gemm_*`. The operator's own pairing - a BF16 +payload with FP32 route weights - is accepted; other combinations must share the +payload dtype, so the oracle's precision is unambiguous. + +Ordering, matching the kernel contract: + +* experts ascend, i.e. the router `offsets` segments; +* combine slots ascend per token, in FP32; +* no reduction crosses token rows. +""" + +from __future__ import annotations + +import torch + +from rl_engine.reference.moe.nemotron_router import dispatch_tokens + +CONTRACT_VERSION = "nemotron-experts-sm90-v1" +# Bound every 32-bit index product: 6 * MAX_TOKENS packed rows, 2688 columns. +MAX_TOKENS = 65536 + + +def _check_stacked(w_up: torch.Tensor, w_down: torch.Tensor, width: int) -> int: + if w_up.ndim != 3 or w_down.ndim != 3: + raise ValueError("stacked expert weights must be rank three") + if w_up.shape[0] != w_down.shape[0]: + raise ValueError("up and down weights must cover the same expert set") + if w_up.shape[2] != width or w_down.shape[1] != width: + raise ValueError("hidden widths must agree") + if w_up.shape[1] != w_down.shape[2]: + raise ValueError("intermediate widths must agree") + if w_up.dtype != w_down.dtype or w_up.device != w_down.device: + raise ValueError("stacked weights must share dtype and device") + return w_up.shape[0] + + +def _check_shared( + width: int, w_sh1: torch.Tensor | None, w_sh2: torch.Tensor | None +) -> tuple[torch.Tensor, torch.Tensor]: + if (w_sh1 is None) != (w_sh2 is None): + raise ValueError("shared expert weights must be supplied as a pair") + if w_sh1 is None or w_sh2 is None: + raise ValueError("shared expert weights are absent") + if w_sh1.ndim != 2 or w_sh2.ndim != 2: + raise ValueError("shared expert weights must be rank two") + if w_sh1.shape[1] != width or w_sh2.shape[0] != width: + raise ValueError("shared expert hidden widths must agree") + if w_sh1.shape[0] != w_sh2.shape[1]: + raise ValueError("shared expert intermediate widths must agree") + return w_sh1, w_sh2 + + +def _leaf_round(value: torch.Tensor, leaf: torch.dtype | None) -> torch.Tensor: + return value if leaf is None else value.to(leaf).to(value.dtype) + + +def routed_experts( + packed: torch.Tensor, + weights: torch.Tensor, + permutation: torch.Tensor, + offsets: torch.Tensor, + w_up: torch.Tensor, + w_down: torch.Tensor, +) -> torch.Tensor: + """Wide FP32/FP64 `[tokens, hidden]` grouped expert output before the exit cast. + + `packed` is the router payload, so this is the split form of + `experts_reference` that a provider takes: the combine sums slots ascending + and never reduces across token rows. + """ + tokens, slots = weights.shape + width = packed.shape[1] + experts, intermediate = w_up.shape[0], w_up.shape[1] + if packed.shape[0] != tokens * slots: + raise ValueError("packed must hold one row per (token, slot)") + compute = packed.dtype + leaf = None if compute == torch.float64 else torch.bfloat16 + if offsets.numel() != experts + 1: + raise ValueError("offsets must hold experts+1 exclusive prefix sums") + if permutation.numel() != packed.shape[0]: + raise ValueError("permutation must cover every packed row") + up = torch.zeros((packed.shape[0], intermediate), dtype=compute, device=packed.device) + for expert in range(experts): + lo = int(offsets[expert]) + hi = int(offsets[expert + 1]) + if lo == hi: + continue # An empty expert contributes no rows and no reduction. + up[lo:hi] = packed[lo:hi] @ w_up[expert].transpose(0, 1) + gate = _leaf_round(up, leaf).relu() + gate = _leaf_round(gate * gate, leaf) + down = torch.zeros((packed.shape[0], width), dtype=compute, device=packed.device) + for expert in range(experts): + lo = int(offsets[expert]) + hi = int(offsets[expert + 1]) + if lo == hi: + continue + down[lo:hi] = gate[lo:hi] @ w_down[expert].transpose(0, 1) + combined = down.index_select(0, permutation.argsort()).reshape(tokens, slots, width) + routed = torch.zeros((tokens, width), dtype=compute, device=packed.device) + for slot in range(slots): + routed = routed + combined[:, slot] * weights[:, slot : slot + 1] + return routed + + +def shared_experts(x: torch.Tensor, w_sh1: torch.Tensor, w_sh2: torch.Tensor) -> torch.Tensor: + """Wide FP32/FP64 `[tokens, hidden]` shared expert output before the exit cast.""" + compute = x.dtype + leaf = None if compute == torch.float64 else torch.bfloat16 + up = x @ w_sh1.transpose(0, 1) + gate = _leaf_round(up, leaf).relu() + gate = _leaf_round(gate * gate, leaf) + return gate @ w_sh2.transpose(0, 1) + + +def experts_reference( + hidden_states: torch.Tensor, + expert_ids: torch.Tensor, + weights: torch.Tensor, + w_up: torch.Tensor, + w_down: torch.Tensor, + w_sh1: torch.Tensor | None = None, + w_sh2: torch.Tensor | None = None, +) -> torch.Tensor: + """Return `[tokens, hidden]` BF16 (or FP64 for FP64 inputs) expert output. + + `hidden_states` is the unweighted token payload, `expert_ids` the ascending + selected experts and `weights` the router weights for those slots. Dispatch + reuses the router reference `dispatch_tokens`, so the packed order and the + `offsets` prefix sums are identical to the operator contract. + """ + if hidden_states.ndim != 2 or expert_ids.ndim != 2: + raise ValueError("hidden_states and expert_ids must be rank two") + if weights.shape != expert_ids.shape: + raise ValueError("weights must match the selected expert shape") + if expert_ids.shape[0] != hidden_states.shape[0]: + raise ValueError("token counts must agree") + if not hidden_states.is_floating_point() or not weights.is_floating_point(): + raise ValueError("hidden states and weights must be floating tensors") + if expert_ids.dtype != torch.int64 or expert_ids.device != hidden_states.device: + raise ValueError("expert_ids must be int64 on the input device") + if weights.device != hidden_states.device: + raise ValueError("weights must live on the input device") + + width = hidden_states.shape[1] + experts = _check_stacked(w_up, w_down, width) + if w_up.device != hidden_states.device: + raise ValueError("stacked weights must live on the input device") + shared = None + if w_sh1 is not None or w_sh2 is not None: + shared = _check_shared(width, w_sh1, w_sh2) + if weights.dtype != hidden_states.dtype and not ( + hidden_states.dtype == torch.bfloat16 and weights.dtype == torch.float32 + ): + # The operator's canonical pairing is a BF16 payload with FP32 route + # weights; anything else must match the payload so the oracle's precision + # is unambiguous (FP64 in, FP64 out). + raise ValueError( + "weights must be float32 with a bfloat16 payload, or match the payload dtype" + ) + if w_up.dtype != hidden_states.dtype: + raise ValueError("hidden states and stacked weights must share a dtype") + + exact = hidden_states.dtype == torch.float64 + compute = torch.float64 if exact else torch.float32 + permutation, offsets, packed = dispatch_tokens(hidden_states, expert_ids, experts) + wide = routed_experts( + packed.to(compute), + weights.to(compute), + permutation, + offsets, + w_up.to(compute), + w_down.to(compute), + ) + if shared is not None: + wide = wide + shared_experts( + hidden_states.to(compute), shared[0].to(compute), shared[1].to(compute) + ) + return wide if exact else wide.to(torch.bfloat16) diff --git a/rl_engine/runtime/registry.py b/rl_engine/runtime/registry.py index 9d339d5d6..bca015380 100644 --- a/rl_engine/runtime/registry.py +++ b/rl_engine/runtime/registry.py @@ -94,6 +94,14 @@ class OpBackend(Enum, metaclass=_KernelEnumMeta): # Nemotron Nano single-device router; no reference fallback in strict dispatch. TRITON_NEMOTRON_ROUTER = "rl_engine.backends.cuda.moe.nemotron_router.NemotronRouterOp" + # Nemotron Nano grouped expert MLP + fixed-order combine (RFC #434). The + # native SM90 provider is CUDA-only; the shared Triton provider is the ROCm + # implementation and the CUDA fallback when the SM90 build flag is absent. + CUDA_NEMOTRON_EXPERTS = "rl_engine.backends.cuda.moe.nemotron_experts.NemotronExpertsOp" + TRITON_NEMOTRON_EXPERTS = ( + "rl_engine.backends.shared.triton.moe.nemotron_experts.TritonNemotronExpertsOp" + ) + # GRPO loss (group reward normalization + clipped surrogate + KL) TRITON_GRPO_LOSS = "rl_engine.kernels.ops.triton.loss.grpo_loss.TritonGRPOLossOp" PYTORCH_GRPO_LOSS = "rl_engine.kernels.ops.pytorch.loss.grpo_loss.NativeGRPOLossOp" @@ -539,6 +547,10 @@ def __init__(self): self._priority_map = { "cuda": { "nemotron_router_dispatch": [OpBackend.TRITON_NEMOTRON_ROUTER], + "moe_experts_combine": [ + OpBackend.CUDA_NEMOTRON_EXPERTS, + OpBackend.TRITON_NEMOTRON_EXPERTS, + ], "logp": [ OpBackend.CUDA_FUSED_LOGP_GENERIC, OpBackend.FLASHINFER, @@ -615,6 +627,9 @@ def __init__(self): ], }, "rocm": { + # Nemotron experts: the shared Triton provider is the only ROCm + # implementation, so the row has no fallback below it. + "moe_experts_combine": [OpBackend.TRITON_NEMOTRON_EXPERTS], "logp": [ OpBackend.ROCM_AITER, OpBackend.TRITON_GENERIC, @@ -1044,6 +1059,8 @@ def get_op(self, op_type: str, device: torch.device | str | None = None) -> Any: platform = self._platform_for_device(device) if op_type == "nemotron_router_dispatch" and platform != "cuda": raise RuntimeError("Nemotron router is qualified only for NVIDIA SM90 CUDA") + if op_type == "moe_experts_combine" and platform not in ("cuda", "rocm"): + raise RuntimeError("Nemotron experts require NVIDIA SM90 CUDA or ROCm") candidates = self._priority_map.get(platform, {}).get(op_type, [OpBackend.PYTORCH_NATIVE]) for backend in candidates: diff --git a/tests/backends/cuda/test_nemotron_experts_cuda.py b/tests/backends/cuda/test_nemotron_experts_cuda.py new file mode 100644 index 000000000..3ab8b8c6d --- /dev/null +++ b/tests/backends/cuda/test_nemotron_experts_cuda.py @@ -0,0 +1,513 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""SM90 experts: arithmetic pinning, contract assertions, registry and graphs. + +Three independent kinds of evidence: + +* `test_matches_det_gemm_bitwise` and the two `*_matches_det_gemm*` tests pin the + kernel to the repository-wide GEMM arithmetic by rebuilding the same output + through `_C.det_gemm_*` and comparing bits; +* `test_core` runs the shared provider assertions (FP64 qualification, batch + invariance, row relocation, repeatability) for both providers; +* the remaining cases cover provider selection, graph capture, autocast and + geometry rejection. +""" + +from __future__ import annotations + +import pytest +import torch + +from tests.backends import nemotron_experts_cases as cases + +pytestmark = [ + pytest.mark.cuda_only, + pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required"), +] + +_SM90 = pytest.mark.skipif( + not torch.cuda.is_available() + or torch.version.hip is not None + or torch.cuda.get_device_capability() != (9, 0), + reason="NVIDIA SM90 required", +) + + +def _native_compiled() -> bool: + from rl_engine.backends.extension import _C, _EXT_AVAILABLE + + return ( + _EXT_AVAILABLE + and hasattr(_C, "nemotron_experts_forward") + and bool(_C.nemotron_experts_compiled()) + ) + + +def _provider(name): + if name == "native": + if not _native_compiled(): + pytest.skip("native SM90 experts extension is not built") + from rl_engine.backends.cuda.moe.nemotron_experts import NemotronExpertsOp + + return NemotronExpertsOp() + pytest.importorskip("triton") + from rl_engine.backends.shared.triton.moe.nemotron_experts import TritonNemotronExpertsOp + + return TritonNemotronExpertsOp() + + +def _segments(case): + spans = case["offsets"].tolist() + for expert in range(case["w_up"].shape[0]): + lo, hi = spans[expert], spans[expert + 1] + if lo != hi: + yield expert, lo, hi + + +@_SM90 +@pytest.mark.parametrize("tokens", [1024, 4096]) +def test_matches_det_gemm_bitwise(tokens): + """Grouped up/down projections are the det_gemm tile tree, bit for bit.""" + if not _native_compiled(): + pytest.skip("native SM90 experts extension is not built") + from rl_engine.backends.extension import _C + + case = cases.build_case(tokens, "cuda", **cases.REAL) + _, up, z, down, _, shared_up, _, _ = _C.nemotron_experts_forward( + case["packed"], + case["x"], + case["weights"], + case["permutation"], + case["offsets"], + case["w_up"], + case["w_down"], + case["w_sh1"], + case["w_sh2"], + ) + checked = 0 + for expert, lo, hi in _segments(case): + packed = case["packed"][lo:hi].contiguous() + expected = _C.det_gemm_fwd_rhs_transposed(packed, case["w_up"][expert].contiguous()) + cases.bits(up[lo:hi], expected) + # The down projection stores the BF16 tree value widened to FP32, and the + # up epilogue materializes its relu-square operand. + gate = torch.relu(up[lo:hi].float()).square().to(torch.bfloat16) + cases.bits(z[lo:hi], gate) + expected_down = _C.det_gemm_fwd_rhs_transposed(gate, case["w_down"][expert].contiguous()) + assert torch.equal(down[lo:hi], expected_down.float()) + checked += 1 + assert checked >= 2 + # The shared expert is the same kernel with one segment over [0, tokens). + cases.bits(shared_up, _C.det_gemm_fwd_rhs_transposed(case["x"], case["w_sh1"])) + + +def _torch_backward_chain(case, up, grad_out): + """dY, dZ and dU rebuilt through det_gemm on the provider's own `up`.""" + from rl_engine.backends.extension import _C + + original = case["permutation"] + slots = case["weights"].shape[1] + weight = case["weights"].reshape(-1)[original] + dy = (grad_out.float()[original // slots] * weight[:, None]).to(torch.bfloat16) + chain = {} + for expert, lo, hi in _segments(case): + dz = _C.det_gemm_fwd(dy[lo:hi].contiguous(), case["w_down"][expert].contiguous()) + scale = 2.0 * torch.relu(up[lo:hi].float()) + chain[expert] = (lo, hi, dy, dz, (dz.float() * scale).to(torch.bfloat16)) + return dy, chain + + +@_SM90 +def test_dpacked_matches_det_gemm_bitwise(): + """dPacked exercises the transposed-weights tile and the relu-square VJP.""" + if not _native_compiled(): + pytest.skip("native SM90 experts extension is not built") + from rl_engine.backends.extension import _C + + provider = _provider("native") + case = cases.build_case(1024, "cuda", **cases.REAL) + grad_out = cases.cotangent_for(case, seed=3) + packed = case["packed"].detach().clone().requires_grad_(True) + out = provider( + packed, + case["x"], + case["weights"], + case["permutation"], + case["offsets"], + case["w_up"], + case["w_down"], + case["w_sh1"], + case["w_sh2"], + ) + d_packed = torch.autograd.grad(out, packed, grad_out)[0] + _, up, _, _, _, _, _, _ = _C.nemotron_experts_forward( + case["packed"], + case["x"], + case["weights"], + case["permutation"], + case["offsets"], + case["w_up"], + case["w_down"], + case["w_sh1"], + case["w_sh2"], + ) + assert out.shape == (1024, cases.REAL["width"]) + _, chain = _torch_backward_chain(case, up, grad_out) + for expert, (lo, hi, _, _, du) in chain.items(): + expected = _C.det_gemm_fwd(du, case["w_up"][expert].contiguous()) + cases.bits(d_packed[lo:hi], expected) + + +@_SM90 +@pytest.mark.parametrize("tokens", [32, 64, 1024]) +def test_weight_gradients_match_det_gemm(tokens): + """dW: transposed operand pair and per-expert segments, leaf aligned. + + Six experts absorb every route, so each segment holds exactly `tokens` rows. + `det_gemm` keeps its SM90 tile tree only while the reduction length is a + multiple of the 32-wide leaf - which `tokens` is - so the comparison is + bitwise. An unaligned segment falls back to `det_gemm`'s scalar kernel, whose + in-leaf order differs from mma, and is covered by the FP64 qualification + instead. + """ + if not _native_compiled(): + pytest.skip("native SM90 experts extension is not built") + from rl_engine.backends.extension import _C + + geometry = dict(cases.REAL) + geometry["experts"] = 6 + case = cases.build_case(tokens, "cuda", **geometry) + provider = _provider("native") + grad_out = cases.cotangent_for(case, seed=5) + leaves = { + name: case[name].detach().clone().requires_grad_(True) + for name in ("packed", "w_up", "w_down") + } + out = provider( + leaves["packed"], + case["x"], + case["weights"], + case["permutation"], + case["offsets"], + leaves["w_up"], + leaves["w_down"], + case["w_sh1"], + case["w_sh2"], + ) + d_w_up, d_w_down = torch.autograd.grad(out, (leaves["w_up"], leaves["w_down"]), grad_out) + _, up, _, _, _, _, _, _ = _C.nemotron_experts_forward( + case["packed"], + case["x"], + case["weights"], + case["permutation"], + case["offsets"], + case["w_up"], + case["w_down"], + case["w_sh1"], + case["w_sh2"], + ) + _, chain = _torch_backward_chain(case, up, grad_out) + assert len(chain) == 6 + for expert, (lo, hi, dy, _, du) in chain.items(): + assert hi - lo == tokens and tokens % 32 == 0 + cases.bits( + d_w_up[expert], + _C.det_gemm_db_transposed(du, case["packed"][lo:hi].contiguous()).t(), + ) + # dW_down reduces against the gated down-projection operand. + gate = torch.relu(up[lo:hi].float()).square().to(torch.bfloat16) + cases.bits( + d_w_down[expert], + _C.det_gemm_db_transposed(dy[lo:hi].contiguous(), gate).t(), + ) + + +@_SM90 +@pytest.mark.parametrize( + "geometry, tokens", + [ + pytest.param(dict(cases.CORE), 17, id="core-unaligned"), + pytest.param(dict(cases.CORE), 129, id="core-tiles"), + pytest.param(dict(cases.REAL), 64, id="nemotron-real"), + ], +) +def test_instruction_paths_agree_bitwise(monkeypatch, geometry, tokens): + """The wgmma path and the mma.sync path are two spellings of one contract. + + Same leaf width, same leaf rounding, same midpoint tree, so every output and + every gradient must match bit for bit on both. `tokens=17` and `tokens=129` + give per-expert segment lengths that are not multiples of the 32-wide leaf, + which is where the weight gradient pads its last reduction tile, and the + Nemotron geometry exercises truncated 128-wide column tiles (`N=1856`). + """ + if not _native_compiled(): + pytest.skip("native SM90 experts extension is not built") + from rl_engine.backends.extension import _C + + case = cases.build_case(tokens, "cuda", **geometry) + cotangent = cases.cotangent_for(case, seed=11) + + # Resolve a fresh provider inside each environment scope, and check that the + # operator really reports the path the test thinks it is exercising: a cached + # selection would turn this into a self-comparison that can never fail. + monkeypatch.setenv("RL_KERNEL_NEMOTRON_EXPERTS_BACKEND", "sm90_v1") + assert _C.nemotron_experts_preferred_path() == "sm90_v1" + reference, reference_grads = cases.run(_provider("native"), case, cotangent) + # The executed path, not the preference: if a launch ever ignored the + # selector (or cached it), this fails instead of comparing one kernel with + # itself. + assert _C.nemotron_experts_last_path() == "sm90_v1" + + monkeypatch.setenv("RL_KERNEL_NEMOTRON_EXPERTS_BACKEND", "sm90_v2") + assert _C.nemotron_experts_preferred_path() == "sm90_v2" + actual, actual_grads = cases.run(_provider("native"), case, cotangent) + assert _C.nemotron_experts_last_path() == "sm90_v2" + + cases.bits(actual, reference) + for label in cases.gradient_names(case): + cases.bits(actual_grads[label], reference_grads[label]) + + +@_SM90 +@pytest.mark.parametrize("name", ["native", "triton"]) +@pytest.mark.parametrize("concentrated", [False, True]) +def test_core(name, concentrated): + cases.run_core_assertions(_provider(name), "cuda", concentrated=concentrated) + + +@_SM90 +@pytest.mark.parametrize("name", ["native", "triton"]) +def test_contract_geometry(name): + """The documented Nemotron Nano shape, including the shared expert.""" + cases.run_core_assertions(_provider(name), "cuda", geometry=cases.REAL) + + +@_SM90 +def test_provider_cross_check(): + cases.cross_provider_report(_provider("native"), _provider("triton"), "cuda") + + +_SMALL = dict(cases.CORE) + + +def _small_case(tokens=17, **overrides): + geometry = dict(_SMALL) + geometry.update(overrides) + return cases.build_case(tokens, "cuda", **geometry) + + +@_SM90 +def test_provider_selection_prefers_native(monkeypatch): + if not _native_compiled(): + pytest.skip("native SM90 experts extension is not built") + from rl_engine.backends.cuda.moe import nemotron_experts as native + from rl_engine.backends.shared.triton.moe.nemotron_experts import TritonNemotronExpertsOp + from rl_engine.runtime.registry import KernelRegistry + + op = KernelRegistry().get_op("moe_experts_combine", device="cuda") + assert isinstance(op, native.NemotronExpertsOp) + assert op.provider_id == "rlkernel.nemotron_experts.sm90.v1" + + class _Uncompiled: + nemotron_experts_forward = staticmethod(lambda *args, **kwargs: None) + + @staticmethod + def nemotron_experts_compiled(): + return False + + monkeypatch.setattr(native, "_C", _Uncompiled()) + # `native._C` is the module global `NemotronExpertsOp` dereferences, and the + # patched object reports the flag as False: if the provider read another + # object, the next lookup would return the native op and the assert fails. + assert native._C.nemotron_experts_compiled() is False + fallback = KernelRegistry().get_op("moe_experts_combine", device="cuda") + assert isinstance(fallback, TritonNemotronExpertsOp) + assert fallback.provider_id == "rlkernel.nemotron_experts.triton.v1" + + +@_SM90 +@pytest.mark.parametrize("capability", [(8, 0), (10, 0)], ids=["pre-sm90", "blackwell"]) +def test_native_provider_rejects_non_sm90_payload_device(monkeypatch, capability): + """The SM90 gate follows the payload device and admits exactly sm_90.""" + if not _native_compiled(): + pytest.skip("native SM90 experts extension is not built") + provider = _provider("native") + case = _small_case() + arguments = ( + case["packed"], + case["x"], + case["weights"], + case["permutation"], + case["offsets"], + case["w_up"], + case["w_down"], + case["w_sh1"], + case["w_sh2"], + ) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *args, **kwargs: capability) + with pytest.raises(RuntimeError, match="requires NVIDIA SM90"): + provider(*arguments) + + +@pytest.mark.parametrize("device", ["cpu", "mps"]) +def test_unsupported_devices_fail_closed(device): + from rl_engine.runtime.registry import KernelRegistry + + with pytest.raises(RuntimeError, match="SM90 CUDA or ROCm"): + KernelRegistry().get_op("moe_experts_combine", device=device) + + +@_SM90 +def test_provider_id_reports_selected_provider(): + from rl_engine.ops.moe.nemotron_experts import nemotron_experts_provider_id + from rl_engine.runtime.registry import kernel_registry + + selected = kernel_registry.get_op("moe_experts_combine", device="cuda") + assert nemotron_experts_provider_id(selected) in ( + "rlkernel.nemotron_experts.sm90.v1", + "rlkernel.nemotron_experts.triton.v1", + ) + assert nemotron_experts_provider_id() == nemotron_experts_provider_id(selected) + + +@_SM90 +@pytest.mark.parametrize("name", ["native", "triton"]) +def test_graph_replay_observes_input_updates(name): + provider = _provider(name) + case = _small_case(tokens=64) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + cases.call(provider, case) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = cases.call(provider, case) + for _ in range(3): + case["packed"].normal_(std=0.5) + case["x"].normal_(std=0.5) + case["weights"].uniform_() + case["w_up"].normal_(std=0.05) + case["w_down"].normal_(std=0.05) + case["w_sh1"].normal_(std=0.05) + case["w_sh2"].normal_(std=0.05) + graph.replay() + expected = cases.call(provider, case) + cases.bits(captured, expected) + + +@_SM90 +@pytest.mark.parametrize("name", ["native", "triton"]) +def test_autocast_invariance(name): + provider = _provider(name) + case = _small_case() + cotangent = cases.cotangent_for(case) + baseline, baseline_grads = cases.run(provider, case, cotangent) + with torch.autocast("cuda", dtype=torch.bfloat16): + actual, actual_grads = cases.run(provider, case, cotangent) + cases.bits(actual, baseline) + for label in cases.gradient_names(case): + cases.bits(actual_grads[label], baseline_grads[label]) + + +@_SM90 +def test_reject_bad_geometry(): + """Input validation is hardware-independent, so it is exercised once.""" + from rl_engine.ops.moe.nemotron_experts import MAX_TOKENS, nemotron_experts + + case = _small_case() + + def run(**overrides): + arguments = dict(case) + arguments.update(overrides) + return nemotron_experts( + arguments["packed"], + arguments["x"], + arguments["weights"], + arguments["permutation"], + arguments["offsets"], + arguments["w_up"], + arguments["w_down"], + arguments.get("w_sh1"), + arguments.get("w_sh2"), + ) + + with pytest.raises(ValueError, match="six rows per token"): + run(packed=case["packed"][:-1]) + with pytest.raises(ValueError, match="weights must be"): + run(weights=case["weights"].double()) + with pytest.raises(ValueError, match="contiguous"): + run(weights=case["weights"].repeat(1, 2)[:, ::2]) + with pytest.raises(ValueError, match="supplied as a pair"): + run(w_sh2=None) + with pytest.raises(ValueError, match="permutation must cover"): + run(permutation=case["permutation"][:-1]) + with pytest.raises(ValueError, match="experts\\+1"): + run(offsets=case["offsets"][:-1]) + with pytest.raises(ValueError, match="at most"): + oversized = torch.zeros((MAX_TOKENS + 1) * 6, 64, device="cuda", dtype=torch.bfloat16) + nemotron_experts( + oversized, + torch.zeros(MAX_TOKENS + 1, 64, device="cuda", dtype=torch.bfloat16), + torch.zeros(MAX_TOKENS + 1, 6, device="cuda", dtype=torch.float32), + torch.zeros((MAX_TOKENS + 1) * 6, device="cuda", dtype=torch.int64), + torch.zeros(129, device="cuda", dtype=torch.int64), + torch.zeros(128, 64, 64, device="cuda", dtype=torch.bfloat16), + torch.zeros(128, 64, 64, device="cuda", dtype=torch.bfloat16), + ) + + +@_SM90 +@pytest.mark.parametrize("name", ["native", "triton"]) +def test_empty_batch(name): + provider = _provider(name) + case = _small_case() + empty = dict(case) + for label, rows in (("packed", 0), ("x", 0), ("weights", 0), ("permutation", 0)): + empty[label] = case[label][:rows] + # A zero-token batch also needs a routing table that describes zero rows, + # otherwise `offsets` still claims the 17-token workload. + empty["offsets"] = torch.zeros_like(case["offsets"]) + weight_names = ["w_up", "w_down"] + (["w_sh1", "w_sh2"] if "w_sh1" in empty else []) + differentiable = ["packed", "x", "weights", *weight_names] + leaves = { + label: empty[label].detach().clone().requires_grad_(label in differentiable) + for label in differentiable + } + out = provider( + leaves["packed"], + leaves["x"], + leaves["weights"], + empty["permutation"], + empty["offsets"], + leaves["w_up"], + leaves["w_down"], + leaves.get("w_sh1"), + leaves.get("w_sh2"), + ) + assert out.shape == (0, cases.CORE["width"]) and out.dtype == torch.bfloat16 + # Zero tokens still has to produce gradients shaped like their inputs, with + # exact zeros for the weights: an expert-parallel rank can hold no tokens and + # must not hand the optimizer an absent (or 1-D) gradient. + names = ["x", *weight_names] + grad_out = torch.zeros((0, cases.CORE["width"]), device="cuda", dtype=torch.bfloat16) + grads = torch.autograd.grad(out, [leaves[name] for name in names], grad_out, allow_unused=True) + for name, gradient in zip(names, grads): + if name in weight_names: + # A weight gradient is nonzero-shaped, so "absent" is never legitimate: + # it must be present, shaped like the weight, and exactly zero. + assert gradient is not None, f"{name} gradient is absent for a zero-token batch" + assert gradient.shape == leaves[name].shape, ( + f"{name} gradient shape {tuple(gradient.shape)} != " + f"input shape {tuple(leaves[name].shape)}" + ) + assert not gradient.any(), f"{name} gradient is not zero" + else: + # A row-shaped gradient is an empty tensor of the input's shape, not + # absent and not a flat `(0,)`; both providers return it. + assert gradient is not None, f"{name} gradient is absent for a zero-token batch" + assert gradient.shape == leaves[name].shape, ( + f"{name} gradient shape {tuple(gradient.shape)} != " + f"input shape {tuple(leaves[name].shape)}" + ) diff --git a/tests/backends/nemotron_experts_cases.py b/tests/backends/nemotron_experts_cases.py new file mode 100644 index 000000000..18506f36c --- /dev/null +++ b/tests/backends/nemotron_experts_cases.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Shared provider assertions for the Nemotron experts operator. + +Imported by the CUDA and ROCm suites so both columns check one contract; the +module name has no `test_` prefix, so pytest never collects it directly. + +Two geometries: `CORE` stays small enough for a fast suite while still crossing +tile boundaries (17 tokens, 8 experts, 256/128 widths), and `REAL` is the +Nemotron Nano shape (2688/1856, 128 experts, shared 3712) used by the +operator-level cases. +""" + +from __future__ import annotations + +import torch + +from rl_engine.reference.moe.nemotron_experts import routed_experts, shared_experts +from rl_engine.reference.moe.nemotron_router import dispatch_tokens, route_scores + +SLOTS = 6 +CORE = {"width": 256, "intermediate": 128, "experts": 8, "shared_width": 384} +REAL = {"width": 2688, "intermediate": 1856, "experts": 128, "shared_width": 3712} +# Empirical thresholds for this workload, not universal error bounds. The floor +# is set by the contract itself: leaves round to BF16 (eps 7.8e-3) and the +# combine adds six signed terms before the single exit cast. Measured relative L2 +# against an unrounded FP64 pipeline: native 7.1e-3 / 8.8e-3 and Triton 5.4e-3 / +# 1.0e-2 (CORE / REAL geometry) for the forward, similar for the gradients. +# Bitwise batch and relocation invariance, not this number, is the operator's +# actual claim. +FORWARD_L2 = 2e-2 +PAYLOAD_L2 = 2e-2 +WEIGHT_L2 = 2e-2 +ROUTE_L2 = 2e-2 +# The two providers agree to ~1.6 BF16 ulp: they share the leaf width, the leaf +# rounding and the exit cast but not the association of the leaf sums, and the +# output is BF16, so a sub-ulp cross-provider bound is not attainable. Measured +# 8.3e-3 on CORE and 1.3e-2 on REAL geometry. +CROSS_PROVIDER_L2 = 2e-2 +GRAD_TARGETS = ("packed", "x", "weights", "w_up", "w_down", "w_sh1", "w_sh2") + + +def bits(actual: torch.Tensor, expected: torch.Tensor) -> None: + assert actual.shape == expected.shape, f"{tuple(actual.shape)} != {tuple(expected.shape)}" + assert torch.equal( + actual.contiguous().view(torch.uint8), expected.contiguous().view(torch.uint8) + ), "bitwise mismatch" + + +def relative_l2(actual: torch.Tensor, expected: torch.Tensor) -> float: + difference = (actual.double() - expected.double()).norm() + return difference.div(expected.double().norm().clamp_min(1e-30)).item() + + +def build_case( + tokens, + device, + *, + width, + intermediate, + experts, + shared_width=None, + seed=434, + concentrated=False, + dtype=torch.bfloat16, +): + """Router output, payload and stacked weights for one workload.""" + generator = torch.Generator().manual_seed(seed) + payload = torch.randn(tokens, width, generator=generator) * 0.5 + scores = torch.rand(tokens, experts, generator=generator) + bias = torch.zeros(experts) + if concentrated: + # Six heavily corrected experts collect every route; the rest are empty. + bias[: min(SLOTS, experts)] = 4.0 + ids, route_weights = route_scores(scores, bias, min(SLOTS, experts)) + permutation, offsets, packed = dispatch_tokens(payload, ids, experts) + case = { + "packed": packed.to(device=device, dtype=dtype).contiguous(), + "x": payload.to(device=device, dtype=dtype).contiguous(), + "weights": route_weights.to(device=device).contiguous(), + "permutation": permutation.to(device=device), + "offsets": offsets.to(device=device), + "w_up": (torch.randn(experts, intermediate, width, generator=generator) * 0.05) + .to(device=device, dtype=dtype) + .contiguous(), + "w_down": (torch.randn(experts, width, intermediate, generator=generator) * 0.05) + .to(device=device, dtype=dtype) + .contiguous(), + "ids": ids, + } + if shared_width is not None: + case["w_sh1"] = ( + (torch.randn(shared_width, width, generator=generator) * 0.05) + .to(device=device, dtype=dtype) + .contiguous() + ) + case["w_sh2"] = ( + (torch.randn(width, shared_width, generator=generator) * 0.05) + .to(device=device, dtype=dtype) + .contiguous() + ) + return case + + +def cotangent_for(case, seed=7): + generator = torch.Generator().manual_seed(seed) + width = case["x"].shape[1] + values = torch.randn(case["x"].shape[0], width, generator=generator) * 0.01 + return values.to(device=case["x"].device, dtype=case["x"].dtype) + + +def call(provider, case): + return provider( + case["packed"], + case["x"], + case["weights"], + case["permutation"], + case["offsets"], + case["w_up"], + case["w_down"], + case.get("w_sh1"), + case.get("w_sh2"), + ) + + +def _leaves(case): + leaves = {} + for name, value in case.items(): + tensor = value.detach().clone() + if name in GRAD_TARGETS: + tensor.requires_grad_(True) + leaves[name] = tensor + return leaves + + +def gradient_names(case): + shared = "w_sh1" in case + return [name for name in GRAD_TARGETS if name in case and (name != "x" or shared)] + + +def run(provider, case, cotangent=None): + """Forward plus, for a fixed cotangent, every input gradient.""" + leaves = _leaves(case) + out = call(provider, leaves) + if cotangent is None: + return out, {} + names = gradient_names(case) + grads = torch.autograd.grad(out, [leaves[name] for name in names], cotangent) + for name, gradient in zip(names, grads): + # A gradient must be shaped like its input: autograd broadcasts a stray + # leading dimension away instead of raising, which would hand callers a + # `[1, ...]` `.grad` for a rank-two parameter. + assert gradient.shape == leaves[name].shape, ( + f"{name} gradient shape {tuple(gradient.shape)} != input " + f"{tuple(leaves[name].shape)}" + ) + return out, dict(zip(names, grads)) + + +def exact(case, cotangent): + """FP64 wide output and gradients from the same inputs, no exit cast.""" + names = gradient_names(case) + leaves = {name: case[name].detach().double().requires_grad_() for name in names} + wide = routed_experts( + leaves["packed"], + leaves["weights"], + case["permutation"], + case["offsets"], + leaves["w_up"], + leaves["w_down"], + ) + if "w_sh1" in case: + wide = wide + shared_experts(leaves["x"], leaves["w_sh1"], leaves["w_sh2"]) + grads = torch.autograd.grad(wide, [leaves[name] for name in names], cotangent.double()) + return wide, dict(zip(names, grads)) + + +def isolated_token(case, token): + """A one-token workload whose packed rows are that token's six copies.""" + ids = case["ids"][token] + slots = int(ids.numel()) + experts = case["w_up"].shape[0] + device = case["x"].device + counts = torch.bincount(ids, minlength=experts) + offsets = torch.cat((counts.new_zeros(1), counts.cumsum(0))).to(device) + permutation = torch.argsort(ids, stable=True).to(device) + row = case["x"][token : token + 1] + isolated = { + "packed": row.expand(slots, -1).contiguous(), + "x": row.clone().contiguous(), + "weights": case["weights"][token : token + 1].clone(), + "permutation": permutation, + "offsets": offsets, + "w_up": case["w_up"], + "w_down": case["w_down"], + "ids": ids.reshape(1, slots), + } + if "w_sh1" in case: + isolated["w_sh1"] = case["w_sh1"] + isolated["w_sh2"] = case["w_sh2"] + return isolated + + +def relocated(case, order): + """The same workload with its token rows moved by `order`.""" + device = case["x"].device + device_order = order.to(case["x"].device) + payload = case["x"].index_select(0, device_order) + ids = case["ids"].index_select(0, order) + weights = case["weights"].index_select(0, device_order) + experts = case["w_up"].shape[0] + permutation, offsets, packed = dispatch_tokens(payload.detach().cpu().float(), ids, experts) + moved = dict(case) + moved.update( + { + "packed": packed.to(device=device, dtype=case["x"].dtype).contiguous(), + "x": payload.contiguous(), + "weights": weights.contiguous(), + "permutation": permutation.to(device=device), + "offsets": offsets.to(device=device), + "ids": ids, + } + ) + return moved + + +def _assert_close(actual, expected, limit, label): + error = relative_l2(actual, expected) + assert error <= limit, f"{label}: relative L2 {error:.3e} > {limit:.3e}" + + +def run_core_assertions(provider, device, geometry=None, *, concentrated=False, seed=434): + """FP64 qualification, batch invariance, relocation and repeatability.""" + geometry = dict(CORE if geometry is None else geometry) + width = geometry["width"] + case = build_case(17, device, seed=seed, concentrated=concentrated, **geometry) + reference = build_case(17, device, seed=seed, concentrated=concentrated, **geometry) + cotangent = cotangent_for(case, seed=seed + 1) + + out, grads = run(provider, case, cotangent) + assert out.dtype == torch.bfloat16 and out.shape == (17, width) + wide, expected = exact(case, cotangent) + _assert_close(out, wide, FORWARD_L2, "forward vs FP64") + _assert_close(grads["packed"], expected["packed"], PAYLOAD_L2, "dPacked vs FP64") + _assert_close(grads["w_up"], expected["w_up"], WEIGHT_L2, "dW_up vs FP64") + _assert_close(grads["w_down"], expected["w_down"], WEIGHT_L2, "dW_down vs FP64") + _assert_close(grads["weights"], expected["weights"], ROUTE_L2, "dWeights vs FP64") + if "w_sh1" in case: + _assert_close(grads["x"], expected["x"], PAYLOAD_L2, "dX vs FP64") + _assert_close(grads["w_sh1"], expected["w_sh1"], WEIGHT_L2, "dW_sh1 vs FP64") + _assert_close(grads["w_sh2"], expected["w_sh2"], WEIGHT_L2, "dW_sh2 vs FP64") + + # A token's rows must not depend on the batch: same content, same bits. + inverse = torch.argsort(case["permutation"]) + for token in sorted({0, 5, 16}): + single = isolated_token(case, token) + one_out, one_grads = run(provider, single, cotangent[token : token + 1]) + bits(out[token], one_out[0]) + _assert_close( + one_grads["weights"], + expected["weights"][token : token + 1], + ROUTE_L2, + "isolated dWeights", + ) + bits(grads["weights"][token], one_grads["weights"][0]) + if "w_sh1" in case: + bits(grads["x"][token], one_grads["x"][0]) + # `slot_order` maps a sorted-expert position to the original route slot; + # the isolated payload is packed in that sorted order, so matching the + # original slot `slot` needs the inverse map (its sorted position), not + # `slot_order[slot]` -- the two agree only for an involution. + slot_order = torch.argsort(case["ids"][token]).to(case["packed"].device) + slot_position = torch.argsort(slot_order) + for slot in range(single["permutation"].numel()): + full_row = inverse[token * slot_order.numel() + slot] + isolated_row = slot_position[slot] + bits(grads["packed"][full_row], one_grads["packed"][isolated_row]) + + # Row relocation: the same rows appear in different tiles, same bits. + order = torch.tensor([16, 3, 11, 0, 7, 14, 2, 9, 5, 12, 1, 15, 8, 4, 13, 6, 10]) + moved_case = relocated(case, order) + device_order = order.to(cotangent.device) + moved_out, moved_grads = run(provider, moved_case, cotangent.index_select(0, device_order)) + bits(out.index_select(0, device_order), moved_out) + bits(grads["weights"].index_select(0, device_order), moved_grads["weights"]) + if "w_sh1" in case: + bits(grads["x"].index_select(0, device_order), moved_grads["x"]) + moved_inverse = torch.argsort(moved_case["permutation"]) + for moved_token in (0, 5, 11): + original = int(order[moved_token]) + for slot in range(SLOTS): + bits( + grads["packed"][inverse[original * SLOTS + slot]], + moved_grads["packed"][moved_inverse[moved_token * SLOTS + slot]], + ) + + # Repeatability: an identical call reproduces every bit. + repeat_out, repeat_grads = run(provider, reference, cotangent) + bits(out, repeat_out) + for name in gradient_names(case): + bits(grads[name], repeat_grads[name]) + + return out, grads + + +def cross_provider_report(provider, other, device, geometry=None, limit=CROSS_PROVIDER_L2): + """Relative L2 between two providers of the same contract (no bitwise claim).""" + geometry = dict(CORE if geometry is None else geometry) + case = build_case(23, device, **geometry) + cotangent = cotangent_for(case, seed=11) + out, grads = run(provider, case, cotangent) + other_out, other_grads = run(other, case, cotangent) + report = {"forward": relative_l2(other_out, out)} + for name in gradient_names(case): + report[name] = relative_l2(other_grads[name], grads[name]) + for label, error in report.items(): + assert error <= limit, f"cross-provider {label}: relative L2 {error:.3e} > {limit:.3e}" + return report diff --git a/tests/backends/rocm/test_nemotron_experts_rocm.py b/tests/backends/rocm/test_nemotron_experts_rocm.py new file mode 100644 index 000000000..97063241a --- /dev/null +++ b/tests/backends/rocm/test_nemotron_experts_rocm.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""ROCm experts: the shared Triton provider against the same contract. + +The ROCm column has no native implementation, so these cases cover the Triton +provider only. They were authored without ROCm hardware; the CUDA suite runs the +same `run_core_assertions` body, so the arithmetic, invariance and gradient +assertions are identical. +""" + +from __future__ import annotations + +import pytest +import torch + +from tests.backends import nemotron_experts_cases as cases + +pytestmark = pytest.mark.skipif( + torch.version.hip is None or not torch.cuda.is_available(), + reason="ROCm device required", +) + + +def _provider(): + pytest.importorskip("triton") + from rl_engine.backends.shared.triton.moe.nemotron_experts import TritonNemotronExpertsOp + + return TritonNemotronExpertsOp() + + +@pytest.mark.parametrize("concentrated", [False, True]) +def test_core(concentrated): + cases.run_core_assertions(_provider(), "cuda", concentrated=concentrated) + + +def test_contract_geometry(): + cases.run_core_assertions(_provider(), "cuda", geometry=cases.REAL) + + +def test_registry_selects_the_shared_provider(): + from rl_engine.backends.shared.triton.moe.nemotron_experts import TritonNemotronExpertsOp + from rl_engine.runtime.registry import KernelRegistry + + op = KernelRegistry().get_op("moe_experts_combine", device="cuda") + assert isinstance(op, TritonNemotronExpertsOp) + assert op.provider_id == "rlkernel.nemotron_experts.triton.v1" + + +def test_provider_id_reports_triton(): + from rl_engine.ops.moe.nemotron_experts import nemotron_experts_provider_id + + assert nemotron_experts_provider_id(_provider()) == "rlkernel.nemotron_experts.triton.v1" diff --git a/tests/reference/test_nemotron_experts_reference.py b/tests/reference/test_nemotron_experts_reference.py new file mode 100644 index 000000000..700ed3b49 --- /dev/null +++ b/tests/reference/test_nemotron_experts_reference.py @@ -0,0 +1,291 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors +"""Contract, cast points and differentiability of the experts reference.""" + +import math + +import pytest +import torch + +from rl_engine.reference.moe.nemotron_experts import MAX_TOKENS, experts_reference + + +def _toy( + seed=434, + tokens=6, + width=8, + intermediate=5, + shared_width=7, + experts=4, + slots=3, + dtype=torch.float64, + weight_dtype=None, +): + generator = torch.Generator().manual_seed(seed) + exact = torch.float64 + x = torch.randn(tokens, width, generator=generator, dtype=exact) + w_up = torch.randn(experts, intermediate, width, generator=generator, dtype=exact) * 0.5 + w_down = torch.randn(experts, width, intermediate, generator=generator, dtype=exact) * 0.5 + w_sh1 = torch.randn( + shared_width, width, generator=torch.Generator().manual_seed(seed + 1), dtype=exact + ) + w_sh2 = torch.randn( + width, shared_width, generator=torch.Generator().manual_seed(seed + 2), dtype=exact + ) + if tokens == 0: + # `torch.stack` rejects an empty list, so the zero-token case must not + # go through the comprehension at all. + ids = torch.zeros((0, slots), dtype=torch.int64) + else: + ids = torch.stack( + [torch.arange(slots) + (index % (experts - slots + 1)) for index in range(tokens)] + ) + weights = torch.rand( + tokens, slots, generator=torch.Generator().manual_seed(seed + 3), dtype=exact + ) + return ( + x.to(dtype), + ids, + weights.to(dtype if weight_dtype is None else weight_dtype), + w_up.to(dtype), + w_down.to(dtype), + w_sh1.to(dtype), + w_sh2.to(dtype), + ) + + +def _fsum_dot(left, right): + return math.fsum(a * b for a, b in zip(left, right)) + + +def _scalar_oracle(x, ids, weights, w_up, w_down, w_sh1=None, w_sh2=None): + """Row-local scalar composition in ascending expert/slot order.""" + tokens, width = x.shape + out = [[0.0] * width for _ in range(tokens)] + for token in range(tokens): + for slot in range(ids.shape[1]): + expert = int(ids[token, slot]) + up = [_fsum_dot(x[token], w_up[expert][index]) for index in range(w_up.shape[1])] + gate = [max(value, 0.0) ** 2 for value in up] + down = [_fsum_dot(gate, w_down[expert][index]) for index in range(width)] + for index in range(width): + out[token][index] += float(weights[token, slot]) * down[index] + if w_sh1 is not None: + up = [_fsum_dot(x[token], w_sh1[index]) for index in range(w_sh1.shape[0])] + gate = [max(value, 0.0) ** 2 for value in up] + down = [_fsum_dot(gate, w_sh2[index]) for index in range(width)] + for index in range(width): + out[token][index] += down[index] + return torch.tensor(out, dtype=torch.float64).reshape(tokens, width) + + +def test_scalar_oracle_toy_geometry(): + x, ids, weights, w_up, w_down, w_sh1, w_sh2 = _toy() + actual = experts_reference(x, ids, weights, w_up, w_down, w_sh1, w_sh2) + expected = _scalar_oracle(x, ids, weights, w_up, w_down, w_sh1, w_sh2) + assert actual.dtype == torch.float64 + torch.testing.assert_close(actual, expected, atol=1e-13, rtol=1e-13) + + +def test_shared_expert_is_additive_and_optional(): + x, ids, weights, w_up, w_down, w_sh1, w_sh2 = _toy() + routed = experts_reference(x, ids, weights, w_up, w_down) + both = experts_reference(x, ids, weights, w_up, w_down, w_sh1, w_sh2) + shared_only = _scalar_oracle( + x, + torch.zeros((x.shape[0], 0), dtype=torch.int64), + weights[:, :0], + w_up, + w_down, + w_sh1, + w_sh2, + ) + torch.testing.assert_close(both - routed, shared_only, atol=1e-13, rtol=1e-13) + with pytest.raises(ValueError): + experts_reference(x, ids, weights, w_up, w_down, w_sh1, None) + + +def test_empty_expert_and_zero_tokens(): + x, ids, weights, w_up, w_down, w_sh1, w_sh2 = _toy(experts=10, slots=2) + base = experts_reference(x, ids, weights, w_up, w_down) + used = torch.zeros(w_up.shape[0], dtype=torch.bool) + used[ids.reshape(-1)] = True + assert int((~used).sum()) >= 2 # The toy must actually leave segments empty. + unused_perturbed = w_up.clone() + unused_perturbed[~used] += 1.0 + assert torch.equal(base, experts_reference(x, ids, weights, unused_perturbed, w_down)) + used_perturbed = w_up.clone() + used_perturbed[used] += 1.0 + assert not torch.equal(base, experts_reference(x, ids, weights, used_perturbed, w_down)) + for shared in ((w_sh1, w_sh2), (None, None)): + empty_out = experts_reference( + torch.zeros(0, x.shape[1], dtype=torch.float64), + torch.zeros((0, ids.shape[1]), dtype=torch.int64), + torch.zeros((0, ids.shape[1]), dtype=torch.float64), + w_up, + w_down, + *shared, + ) + assert empty_out.shape == (0, x.shape[1]) + assert empty_out.dtype == torch.float64 + + +def test_single_expert_matches_row_local_oracle(): + x, ids, weights, w_up, w_down, _, _ = _toy(experts=2, slots=1) + actual = experts_reference(x, ids, weights, w_up, w_down) + expected = _scalar_oracle(x, ids, weights, w_up, w_down) + torch.testing.assert_close(actual, expected, atol=1e-13, rtol=1e-13) + + +def _cast_point_error(payload_dtype, weight_dtype=None): + """Relative L2 of the reference against the scalar oracle for one pairing.""" + x, ids, weights, w_up, w_down, w_sh1, w_sh2 = _toy( + dtype=payload_dtype, weight_dtype=weight_dtype + ) + actual = experts_reference(x, ids, weights, w_up, w_down, w_sh1, w_sh2) + exact = _scalar_oracle( + x.double(), + ids, + weights.double(), + w_up.double(), + w_down.double(), + w_sh1.double(), + w_sh2.double(), + ) + assert actual.dtype == torch.bfloat16 + return (actual.double() - exact).norm() / exact.norm() + + +@pytest.mark.parametrize( + "payload_dtype, weight_dtype", + [ + pytest.param(torch.bfloat16, torch.float32, id="canonical-bf16-fp32-routes"), + pytest.param(torch.bfloat16, None, id="bf16-payload-bf16-routes"), + pytest.param(torch.float32, torch.float32, id="fp32-payload"), + ], +) +def test_cast_points_round_leaves_and_exit_only(payload_dtype, weight_dtype): + error = _cast_point_error(payload_dtype, weight_dtype) + # The BF16 rounding points (up result, gate, exit cast) are dtype-independent: + # the payload dtype only decides the matmul precision, so every accepted + # pairing sits in the same 1e-3 band. An unrounded path would be at 1e-13 + # (that is what the FP64 cases assert), hence the two-sided bound. + assert 1e-5 < error < 5e-3 + + +def test_payload_dtype_does_not_move_the_rounding_band(): + """BF16 and FP32 payloads agree to well within the bound, unlike FP64. + + Measured on the toy geometry: FP32 payload 3.1e-3, BF16 payload 2.4e-3, FP64 + payload 1e-13. A change that dropped a `_leaf_round` would move a BF16/FP32 + case towards the FP64 figure, so the band is what pins the cast points. + """ + fp32_payload = _cast_point_error(torch.float32, torch.float32) + bf16_payload = _cast_point_error(torch.bfloat16, torch.float32) + assert max(fp32_payload, bf16_payload) < 3 * min(fp32_payload, bf16_payload) + x, ids, weights, w_up, w_down, w_sh1, w_sh2 = _toy() + exact = _scalar_oracle(x, ids, weights, w_up, w_down, w_sh1, w_sh2) + fp64 = experts_reference(x, ids, weights, w_up, w_down, w_sh1, w_sh2).double() + assert (fp64 - exact).norm() / exact.norm() < 1e-12 + + +def test_rejects_a_payload_weight_dtype_mismatch(): + x, ids, weights, w_up, w_down, w_sh1, w_sh2 = _toy(dtype=torch.bfloat16) + with pytest.raises(ValueError, match="share dtype and device"): + experts_reference(x, ids, weights, w_up.double(), w_down, w_sh1, w_sh2) + with pytest.raises(ValueError, match="weights must be float32"): + experts_reference(x, ids, weights.double(), w_up, w_down, w_sh1, w_sh2) + + +def test_token_permutation_invariance(): + x, ids, weights, w_up, w_down, w_sh1, w_sh2 = _toy(tokens=9) + order = torch.tensor([8, 0, 5, 2, 7, 1, 6, 4, 3]) + base = experts_reference(x, ids, weights, w_up, w_down, w_sh1, w_sh2) + moved = experts_reference( + x.index_select(0, order), + ids.index_select(0, order), + weights.index_select(0, order), + w_up, + w_down, + w_sh1, + w_sh2, + ) + # The reference accumulates with torch.matmul, whose per-row reduction is + # not row-order stable; the operator's row-local kernels are (tests on GPU). + torch.testing.assert_close(base.index_select(0, order), moved, atol=1e-12, rtol=1e-12) + + +def test_expert_id_mapping_is_not_shared_between_experts(): + x, ids, weights, w_up, w_down, w_sh1, w_sh2 = _toy() + swapped = w_up.flip(0) + base = experts_reference(x, ids, weights, w_up, w_down) + other = experts_reference(x, ids, weights, swapped, w_down) + assert not torch.equal(base.to(torch.float32), other.to(torch.float32)) + assert torch.equal(experts_reference(x, ids, weights, w_up, w_down), base) + + +def test_gradcheck_weights_and_payload(): + x, ids, weights, w_up, w_down, _, _ = _toy( + tokens=2, width=3, intermediate=2, experts=2, slots=2 + ) + x = x.clone().requires_grad_() + w_up = w_up.clone().requires_grad_() + w_down = w_down.clone().requires_grad_() + assert torch.autograd.gradcheck( + lambda hidden, up, down: experts_reference(hidden, ids, weights, up, down), + (x, w_up, w_down), + ) + weights_grad = weights.clone().requires_grad_() + assert torch.autograd.gradcheck( + lambda route: experts_reference(x.detach(), ids, route, w_up.detach(), w_down.detach()), + (weights_grad,), + ) + + +def test_shared_expert_is_differentiable(): + x, ids, weights, w_up, w_down, w_sh1, w_sh2 = _toy( + tokens=2, width=3, intermediate=2, shared_width=2, experts=2, slots=2 + ) + x = x.clone().requires_grad_() + w_sh1 = w_sh1.clone().requires_grad_() + assert torch.autograd.gradcheck( + lambda hidden, up, down: experts_reference(hidden, ids, weights, w_up, w_down, up, down), + (x, w_sh1, w_sh2.clone().requires_grad_()), + ) + + +def test_weights_scale_each_route_only(): + x, ids, weights, w_up, w_down, _, _ = _toy(slots=2, experts=2) + weights = weights.clone() + weights[:, 0] = 0.0 + actual = experts_reference(x, ids, weights, w_up, w_down) + expected = _scalar_oracle(x, ids, weights, w_up, w_down) + torch.testing.assert_close(actual, expected, atol=1e-13, rtol=1e-13) + + +@pytest.mark.parametrize( + "mutation,message", + [ + (lambda tensors: (tensors[0][:, :-1],) + tensors[1:], "widths must agree"), + ( + lambda tensors: tensors[:2] + + (tensors[2], tensors[3], tensors[4][:2], tensors[5], tensors[6]), + "same expert set", + ), + ( + lambda tensors: tensors[:1] + (tensors[1].to(torch.int32),) + tensors[2:], + "int64 on the input device", + ), + ], +) +def test_reject_mismatched_geometry(mutation, message): + x, ids, weights, w_up, w_down, w_sh1, w_sh2 = _toy() + mutated = mutation((x, ids, weights, w_up, w_down, w_sh1, w_sh2)) + with pytest.raises(ValueError, match=message): + experts_reference(mutated[0], mutated[1], mutated[2], mutated[3], mutated[4]) + with pytest.raises(ValueError): + experts_reference(x, ids, weights, w_up, w_down, w_sh1, None) + + +def test_token_bound_is_documented(): + assert MAX_TOKENS == 65536