Skip to content
Draft
Show file tree
Hide file tree
Changes from 57 commits
Commits
Show all changes
161 commits
Select commit Hold shift + click to select a range
1604db4
p5 starter
KJLdefeated Sep 1, 2026
b95ba80
ci: fix black line-length drift in vllm_runtime/flash_attn; pin black…
KJLdefeated Sep 1, 2026
e6373c2
[DSv4][P1-S0] Start kit for the P1 work package (mHC + RMSNorm)
zhangj1an Sep 3, 2026
3e836d4
[DSv4][P1-S0] Pin unfused as canonical; fix alpha to Megatron's 3 sca…
zhangj1an Sep 3, 2026
48bf9eb
[DSv4][P1-S0] Correct the P1 task split to match issue #2
zhangj1an Sep 3, 2026
39c4aea
[DSv4][P1-S0] Use the real sub-issues of #2 (#14-#21)
zhangj1an Sep 3, 2026
0a9c889
[DSv4][P5-5] Shared Expert MLP: strict CUDA + Triton kernels (#64)
jyizheng Sep 4, 2026
42aa927
fix(p5-5): use libdevice exp in the Triton SwiGLU core
jyizheng Sep 4, 2026
d1ace11
fix(p5-5): source Triton sigmoid from torch.sigmoid
jyizheng Sep 4, 2026
bef20ee
fix(p5-5): forbid FMA contraction in the Triton kernels via libdevice…
jyizheng Sep 4, 2026
e4f8c14
feat: add deterministic clamp swiglu weighted
Sep 6, 2026
4c7f1b3
fix: expose optional p_s in P5 bindings
Sep 6, 2026
b0784e6
fix: register P5 forward binding
Sep 6, 2026
0e88933
perf: recompute P5 SwiGLU intermediates in backward
Sep 7, 2026
85a3211
perf: optimize P5 SwiGLU state and packed layout
Sep 7, 2026
da6b81b
perf(p5-5): det_gemm / tl.dot performance profiles alongside the stri…
jyizheng Sep 9, 2026
7abd313
test(p5-5): per-profile oracle tolerances for the performance backends
jyizheng Sep 9, 2026
7a491ff
feat(moe): P5.4 MXFP8/MXFP4 grouped GEMM kernel
Bignonia7 Sep 10, 2026
fd2a631
Merge pull request #389 from hsiuyee/feat/p5-task2-clamp-swiglu-weighted
KJLdefeated Sep 12, 2026
ec3a781
Merge branch 'dsv4-p5-dev' into ajiao/p5-5-shared-expert-mlp
KJLdefeated Sep 12, 2026
5d1d6fb
Merge pull request #387 from jyizheng/ajiao/p5-5-shared-expert-mlp
KJLdefeated Sep 12, 2026
06bf98d
[DSv4][P5-3] shared_grouped_lora_delta: three provider backends
ScottNi0809 Sep 16, 2026
6231d2b
[DSv4][P5-3] benchmark: torch-native vs Triton vs CUDA
ScottNi0809 Sep 16, 2026
be2eef8
[DSv4][P5-3] tests and invariance sweep for both deterministic backends
ScottNi0809 Sep 17, 2026
5d3d54e
feat(moe): add P5-4 SM90 WGMMA compute kernel
Bignonia7 Sep 17, 2026
b9570be
feat(p5): mxfp8_act_quant CUDA + Triton kernels, bit-exact with the P…
ryankert01 Sep 18, 2026
fa6a287
test(p5): assert mxfp8_act_quant byte-equality at production shapes a…
ryankert01 Sep 18, 2026
ea76186
refactor(p5): simplify the mxfp8_act_quant wrappers, providers, and b…
ryankert01 Sep 18, 2026
273f486
Merge pull request #422 from Bignonia7/dev5
KJLdefeated Sep 19, 2026
e74af0d
fix(p5): tolerate a missing envs module in the CUDA mxfp8_act_quant w…
ryankert01 Sep 19, 2026
c17f921
test(p5): generate the 2**31-element input directly in bf16
ryankert01 Sep 19, 2026
344f676
[DSv4][P5-4] MXFP8×MXFP4 Routed Expert grouped GEMM: wgmma triton kernel
Bignonia7 Sep 20, 2026
74ae44d
Merge pull request #419 from ryankert01/p5-1-mxfp8-act-quant
KJLdefeated Sep 21, 2026
da5ba25
Merge pull request #416 from ScottNi0809/p5-3-shared-grouped-lora-delta
KJLdefeated Sep 24, 2026
b1fb7ed
Merge branch 'dsv4-p5-dev' into p5-4-mxfp8-mxfp4-grouped-gemm
KJLdefeated Sep 24, 2026
78813ea
Merge pull request #399 from Bignonia7/p5-4-mxfp8-mxfp4-grouped-gemm
KJLdefeated Sep 24, 2026
59bef32
moe grouped gemm
KJLdefeated Sep 24, 2026
dfc0e71
[DSv4][P1-2] fp32_gemm_rms
Sep 24, 2026
4788564
dsv4-refactor
KJLdefeated Sep 24, 2026
9133d68
add fused moe mlp & triton
KJLdefeated Sep 29, 2026
f7560d2
moe triton kernel dsv4 shared/routed fc1+swiglu, fc3
KJLdefeated Oct 3, 2026
6565815
Rebase MHC H Aggregate onto dsv4-p1-dev, as during the time of writin…
zhangj1an Oct 6, 2026
f988217
update sum order for dPRE to be ascending order rather than warp tree
zhangj1an Oct 7, 2026
f47158e
Merge branch 'dsv4-p5-dev' into p5-4-wgmma-triton
Bignonia7 Oct 8, 2026
f082687
Complete MHC h_aggregate P1 integration with Triton backend
nodeeeeee Oct 12, 2026
029d95c
Merge pull request #438 from Bignonia7/p5-4-wgmma-triton
KJLdefeated Oct 19, 2026
2f1376f
Merge dsv4 dev P1-P7 branch to one dev branch
KJLdefeated Oct 30, 2026
9c58c64
Merge pull request #366 from nodeeeeee/main
Flink-ddd Oct 30, 2026
5874b90
Merge pull request #443 from Junyifannnn/fable/dsv4-p1-2-fp32-gemm-rms
KJLdefeated Oct 30, 2026
5b99c55
Merge test-dsv4-cuda into dsv4-p1-dev and resolve layout conflicts
Flink-ddd Oct 30, 2026
a2132bb
Merge pull request #383 from RL-Align/dsv4-p1-dev
Flink-ddd Oct 30, 2026
88afc0b
Merge test-dsv4-cuda into dsv4-p5-dev and resolve layout conflicts
claude Oct 30, 2026
18216a4
Merge pull request #368 from RL-Align/dsv4-p5-dev
KJLdefeated Oct 30, 2026
830b902
merge from p1=p
KJLdefeated Oct 30, 2026
4f0d9df
Merge branch 'main' into test-dsv4-cuda
maxiaosong1124 Oct 30, 2026
1dd053f
Merge branch 'main' into test-dsv4-cuda
Flink-ddd Oct 30, 2026
7123ed8
feat(p7-t02): add sampling-params normalization and fail-closed gate
Endeavor-19 Nov 1, 2026
3a6275f
feat(p7-t02): add M10 input schema, pre-checks and stub provider
Endeavor-19 Nov 1, 2026
289fc9f
feat(p7-t02): wire M10 scoring through the strict linear_logp and cap…
Endeavor-19 Nov 1, 2026
5701b13
docs(p7-t02): add sampling and M10 implementation contract
Endeavor-19 Nov 1, 2026
8a57bf4
test(p7-t02): run sampling and M10 suites in CI
Endeavor-19 Nov 1, 2026
e25127b
fix(p7-t02): close automated review findings in the gate and M10 pre-…
Endeavor-19 Nov 1, 2026
806d9db
fix(p7-t02): harden M10 versioning, provenance and output guards
Endeavor-19 Nov 1, 2026
ae67c6f
fix(p7-t02): make malformed records fail closed and pin layer provenance
Endeavor-19 Nov 1, 2026
437191b
fix(p7-t02): tighten fallback validation and align scoring result views
Endeavor-19 Nov 1, 2026
31ca466
fix(p7-t02): validate fallback entries and record weight-lock verific…
Endeavor-19 Nov 1, 2026
ce9e3a0
fix(p7-t02): require explicit sampling fields and guard raw byte order
Endeavor-19 Nov 1, 2026
02823be
fix(p7-t02): reject non-canonical layer values and unknown fallback n…
Endeavor-19 Nov 1, 2026
a013c9f
fix(p7-t02): require real vocab size and reject untrusted provenance
Endeavor-19 Nov 1, 2026
2ad56da
fix(p7-t02): canonicalize direct layers and narrow gate status mapping
Endeavor-19 Nov 1, 2026
26337b6
Merge branch 'main' into test-dsv4-cuda
inaniloquentee Nov 2, 2026
d28a3e3
Merge branch 'test-dsv4-cuda' into feat/p7-t02-sampling-m10-contract
Flink-ddd Nov 2, 2026
b8536f5
Merge pull request #648 from Endeavor-19/feat/p7-t02-sampling-m10-con…
maxiaosong1124 Nov 2, 2026
ca5a42d
add design docs
KJLdefeated Nov 5, 2026
2b6f595
Merge branch 'test-dsv4-cuda' of https://github.com/RL-Align/RL-Kerne…
KJLdefeated Nov 5, 2026
c2de274
style: apply pre-commit formatting across the DSv4 branch
KJLdefeated Nov 9, 2026
900f01d
fix(dsv4): resolve Phase 0 blockers for the CUDA SM90 path
KJLdefeated Nov 9, 2026
9df841a
Merge origin/main into test-dsv4-cuda
KJLdefeated Nov 9, 2026
d1bfa2b
feat(dsv4): probe DeepGEMM batch invariance and settle D3 (Phase 1)
KJLdefeated Nov 9, 2026
4d6e033
feat(dsv4): probe DeepGEMM FP8 backward kernels
KJLdefeated Nov 9, 2026
06e68f7
feat(dsv4): add per-token-group FP8 activation quantization for DeepGEMM
KJLdefeated Nov 9, 2026
5252ccb
feat(dsv4): pin DeepGEMM and add the DeepGEMM FP8 shared-expert provider
KJLdefeated Nov 9, 2026
ec257d4
feat(dsv4): add CUDA kernels for the shared expert's clamped SwiGLU
KJLdefeated Nov 9, 2026
5fbf9b1
docs(dsv4): add CLAUDE.md handoff notes for the DSv4 CUDA work
KJLdefeated Nov 9, 2026
02c8bd5
docs(dsv4): record Phase 0/4 status and the P5 shared-expert clamp de…
KJLdefeated Nov 9, 2026
ebc4f61
fix(dsv4): do not launch det_gemm kernels for empty shapes
KJLdefeated Nov 9, 2026
988538b
feat(dsv4): add a general DeepGEMM FP8 linear for the dense projections
KJLdefeated Nov 9, 2026
9380cee
feat(dsv4): add the indexer's stable per-row top-k (P2 T04)
KJLdefeated Nov 9, 2026
092024a
feat(dsv4): probe DeepGEMM's FP8 MQA-logits kernels for batch invariance
KJLdefeated Nov 10, 2026
895252a
feat(dsv4): add the indexer query RoPE + FP8 quantization (P2 T03)
KJLdefeated Nov 10, 2026
8b0ef05
feat(dsv4): probe DeepGEMM's BF16 GEMM for the unquantized projections
KJLdefeated Nov 10, 2026
c741be5
feat(dsv4): add the indexer's KV selection on DeepGEMM MQA logits
KJLdefeated Nov 10, 2026
72dd311
docs(dsv4): record the T02 compressor analysis and the parity decision
KJLdefeated Nov 10, 2026
1c90987
docs(dsv4): record the T02 compressor and T05 FlashMLA decisions
KJLdefeated Nov 10, 2026
345cb9e
feat(dsv4): add the indexer KV compressor (P2 T02, head 128)
KJLdefeated Nov 10, 2026
fd3f061
feat(dsv4): add the head-512 main-KV compressor (C4 and C128)
KJLdefeated Nov 10, 2026
1aa0652
feat(dsv4): pin FlashMLA and probe its SM90 sparse kernel (T05)
KJLdefeated Nov 10, 2026
d131e98
feat(dsv4): add the sparse attention core on FlashMLA (T05)
KJLdefeated Nov 10, 2026
e9b4745
feat(dsv4): add the attention query and output side kernels
KJLdefeated Nov 10, 2026
2742396
feat(dsv4): compose the batch-invariant DSv4 attention layer (T06, T07)
KJLdefeated Nov 10, 2026
b5b6d67
perf(dsv4): move the attention glue to CUDA and make decode graph-cap…
KJLdefeated Nov 10, 2026
497b6d9
build(dsv4): move to the vLLM fork's torch 2.13 / CUDA 13 stack
KJLdefeated Nov 10, 2026
690068a
docs(dsv4): record the verl / Megatron / vLLM integration findings
KJLdefeated Nov 10, 2026
b785c36
feat(dsv4): add the batch-invariant MoE router (P3)
KJLdefeated Nov 10, 2026
13b4513
feat(dsv4): add the MoE route plan, fixed-order combine and the EP=1 …
KJLdefeated Nov 10, 2026
f7dc3d1
feat(dsv4): EP MoE block with backward, SM90 routed backward, faster …
KJLdefeated Nov 10, 2026
d21e002
feat(dsv4): add the batch-invariant mHC with fused RMSNorm and dX bac…
KJLdefeated Nov 10, 2026
a12b6b2
feat(dsv4): assemble the DSv4 decoder layer and model forward
KJLdefeated Nov 10, 2026
78254e8
fix(dsv4): use plain RoPE on the sliding-window-only layers
KJLdefeated Nov 10, 2026
3f0ca16
feat(dsv4): add the batch-invariant selected-token logprob head (P7)
KJLdefeated Nov 10, 2026
d87db23
docs(dsv4): tick the indexer op composition (T03)
KJLdefeated Nov 10, 2026
37a3303
feat(dsv4): check P5's scale-residual range on the real FP4 experts
KJLdefeated Nov 10, 2026
ace607e
perf(dsv4): build the MoE route plan with a CUDA counting sort
KJLdefeated Nov 10, 2026
ddbade7
perf(dsv4): stage the mHC gemm_bwd rows with cp.async
KJLdefeated Nov 11, 2026
eb4d8d3
fix(dsv4): return FlashMLA's LSE, and propose the attention backward
KJLdefeated Nov 11, 2026
7389af8
docs(dsv4): add the per-operator performance table
KJLdefeated Nov 11, 2026
eb1f64e
perf(dsv4): run the per-group wo_a GEMMs concurrently at decode sizes
KJLdefeated Nov 11, 2026
86aff0d
feat(dsv4): wrap cuDNN's SM90 sparse attention backward (non-determin…
KJLdefeated Nov 11, 2026
822c868
feat(dsv4): add the attention layer backward (dL/dhidden, non-determi…
KJLdefeated Nov 11, 2026
db2485a
feat(dsv4): add LoRA adapters on the attention projections
KJLdefeated Nov 11, 2026
f3042b0
feat(dsv4): add LoRA to the MoE block (routed + shared expert)
KJLdefeated Nov 11, 2026
9a58311
feat(dsv4): add the full-model LoRA training forward / backward
KJLdefeated Nov 11, 2026
6c82a9e
feat(dsv4): MoE LoRA with expert parallelism
KJLdefeated Nov 11, 2026
b9eea9d
perf(dsv4): fuse the attention backward's RoPE / RMSNorm steps into o…
KJLdefeated Nov 11, 2026
3da74e1
perf(dsv4): move the attention compressor backward to one CUDA kernel
KJLdefeated Nov 11, 2026
4647df0
docs(dsv4): tick the finished design-doc items
KJLdefeated Nov 11, 2026
d3d6f08
perf(dsv4): run the grouped wo_a as one DeepGEMM fp8_einsum below 819…
KJLdefeated Nov 11, 2026
d7938d7
perf(dsv4): cache hc_head's padded mHC weight instead of padding per …
KJLdefeated Nov 11, 2026
892a250
refactor(dsv4): drop the P5 fused single-launch MoE path
KJLdefeated Nov 11, 2026
5804978
feat(dsv4): deferred MoE finite check, graph-capturable decode step
KJLdefeated Nov 11, 2026
a3b509b
docs(dsv4): write the SM90 batch-invariant execution profile
KJLdefeated Nov 11, 2026
b7bfeec
test(dsv4): probe DeepGEMM grouped BF16 GEMMs for per-expert MoE LoRA
KJLdefeated Nov 11, 2026
70f9d85
feat(dsv4): single-device layerwise exact-zero gate
KJLdefeated Nov 11, 2026
c6e6512
docs(dsv4): record the per-expert LoRA and chunked-training decisions
KJLdefeated Nov 11, 2026
8ec3d0a
Merge branch 'main' into test-dsv4-cuda
KJLdefeated Nov 11, 2026
e6300a9
style(dsv4): black the P5 MoE test and benchmark after the fused-path…
KJLdefeated Nov 11, 2026
d8908b5
chore: remove the P7-T05 Megatron training provider and its test doubles
KJLdefeated Nov 11, 2026
3850757
chore: remove the P2 / P6 start kits
KJLdefeated Nov 11, 2026
d85069a
chore: drop tests and benchmarks of legacy P1 / P5 code DSv4 does not…
KJLdefeated Nov 11, 2026
11ac30a
fix: make mypy pass on rl_engine (CI linting)
KJLdefeated Nov 11, 2026
14cf80b
docs(dsv4): one design doc for handoff; operator pages point to it
KJLdefeated Nov 11, 2026
1352359
docs(dsv4): note the unrun EP=2 LoRA check and the current regression…
KJLdefeated Nov 11, 2026
fd0e60d
perf(dsv4): prebuilt token-block map for the SM90 grouped MoE GEMMs
KJLdefeated Nov 12, 2026
6db09cc
test(dsv4): EP = 2 with LoRA: forward bitwise, LoRA dX to rounding
KJLdefeated Nov 15, 2026
7d0b7d9
feat(dsv4): guide and scripts for a verl + vLLM-BI end-to-end test on…
KJLdefeated Nov 15, 2026
0469a3e
feat(dsv4): 200-step run (T4) in the verl end-to-end guide, for 8 x H100
KJLdefeated Nov 15, 2026
5598067
docs(dsv4): shorten the verl end-to-end guide to T1, rollout speed, T4
KJLdefeated Nov 15, 2026
33b65bb
docs(dsv4): rename the verl guide's training run T4 to T2
KJLdefeated Nov 15, 2026
5ae50f3
feat(dsv4): vLLM rollout plugin, bitwise equal to Dsv4Model (no LoRA)
KJLdefeated Nov 15, 2026
f74765a
fix(dsv4): rollout compressors on RL-Kernel kernels; vLLM gate passes…
KJLdefeated Nov 15, 2026
604efcc
docs(dsv4): the vLLM rollout plugin, what it replaces and why; verl s…
KJLdefeated Nov 15, 2026
bdda3f9
perf(dsv4): keep the MoE forward's E4M3 weight expansion across calls
KJLdefeated Nov 16, 2026
7d45e6f
feat(dsv4): verl engine backend "rlkernel" for the DSv4 LoRA actor
KJLdefeated Nov 16, 2026
7b5ccff
feat(dsv4): verl GRPO on RL-Kernel end to end, rollout bitwise equal …
KJLdefeated Nov 16, 2026
e6a01f6
docs(dsv4): verl integration (section 10.1) and status
KJLdefeated Nov 16, 2026
b410ac0
feat(dsv4): workload knobs in the RL-Kernel verl launcher
KJLdefeated Nov 16, 2026
8b83f07
docs(dsv4): 8-GPU guide, RL-Kernel + verl (BI, LoRA) vs verl #7483
KJLdefeated Nov 16, 2026
682fdde
docs(dsv4): CLAUDE.md from the working notes; status of the 8-GPU guide
KJLdefeated Nov 16, 2026
ccf0c66
style(dsv4): black / isort on the vLLM plugin and its gate
KJLdefeated Nov 16, 2026
f7c208b
feat(dsv4): non-invariant rollout switch for the BI ablation
KJLdefeated Nov 16, 2026
bfc8fb3
docs(dsv4): the BI ablation in the 8-GPU guide, design doc and CLAUDE.md
KJLdefeated Nov 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,22 @@ jobs:
tests/backends/rocm/test_rocm_paged_workspace.py \
tests/distributed/collectives/test_collective_ipc_errors.py

- name: Run P7 T05 Megatron Training Readback Tests (CPU-safe)
run: |
PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -q -rs \
tests/test_megatron_training_provider.py \
tests/test_megatron_training_readback.py \
tests/test_megatron_training_readonly.py \
tests/test_megatron_training_gate.py \
tests/test_megatron_training_recorded.py \
tests/test_megatron_training_evidence.py \
tests/test_megatron_training_upstream.py \
tests/test_megatron_training_export.py \
tests/test_megatron_training_gradients.py \
tests/test_megatron_training_ranks.py \
tests/test_megatron_training_gradient_ranks.py \
tests/test_megatron_training_pairing.py

docs:
runs-on: ubuntu-latest
steps:
Expand Down
43 changes: 43 additions & 0 deletions .github/workflows/p2-start-kit.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: P2 start-kit CPU conformance

on:
pull_request:
paths:
- "rl_engine/p2/**"
- "tests/p2/**"
- "scripts/p2_distributed_smoke.py"
- ".github/workflows/p2-start-kit.yml"
workflow_dispatch:

permissions:
contents: read

jobs:
cpu:
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
matrix:
python: ["3.10", "3.12"]
env:
OMP_NUM_THREADS: "1"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- run: python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
- run: python -m pip install numpy pytest ruff
- run: python -m ruff check rl_engine/p2 tests/p2 scripts/p2_distributed_smoke.py
- run: python -m ruff format --check rl_engine/p2 tests/p2 scripts/p2_distributed_smoke.py
- run: python -m pytest tests/p2 -q --junitxml=p2-results.xml
- run: python -m rl_engine.p2 conformance --tokens 513 --output p2-artifact
- run: python -m rl_engine.p2 verify p2-artifact
- uses: actions/upload-artifact@v4
if: always()
with:
name: p2-start-kit-py${{ matrix.python }}
path: |
p2-results.xml
p2-artifact/
retention-days: 14
43 changes: 43 additions & 0 deletions .github/workflows/p6-start-kit.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: P6 start-kit CPU conformance

on:
pull_request:
paths:
- "rl_engine/p6/**"
- "tests/p6/**"
- "docs/archive/design/p6-start-kit.md"
- ".github/workflows/p6-start-kit.yml"
workflow_dispatch:

permissions:
contents: read

jobs:
cpu:
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
matrix:
python: ["3.10", "3.12"]
env:
OMP_NUM_THREADS: "1"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- run: python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
- run: python -m pip install numpy pytest ruff
- run: python -m ruff check rl_engine/p6 tests/p6
- run: python -m ruff format --check rl_engine/p6 tests/p6
- run: python -m pytest tests/p6 -q --junitxml=p6-results.xml
- run: python -m rl_engine.p6 conformance --device torch-cpu --hidden-size 4096 --output p6-artifact
- run: python -m rl_engine.p6 verify p6-artifact
- uses: actions/upload-artifact@v4
if: always()
with:
name: p6-start-kit-py${{ matrix.python }}
path: |
p6-results.xml
p6-artifact/
retention-days: 14
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,9 @@ dmypy.json
# Cython debug symbols
cython_debug/

# Vscode
.vscode

# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
Expand Down
153 changes: 153 additions & 0 deletions benchmarks/benchmark_fp32_gemm_rms.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# Copyright (c) 2026 RL-Kernel Contributors
"""Benchmark P1-2 fp32_gemm_rms: torch-native vs strict Triton vs strict CUDA.

torch-native is the unconstrained reference (cuBLAS matmul + fused torch RMS,
free to use FMA, split-K and library reduction trees): a different arithmetic
contract, reported for context, never an acceptance gate for the strict
backends. Timings are CUDA-event medians over the full autograd entry
(forward, and forward + backward with fixed upstream gradients).

Example:
python benchmarks/benchmark_fp32_gemm_rms.py --tokens 1,16,128,512 \
--k 16384 --warmup 5 --iterations 20 --json out.json
"""

from __future__ import annotations

import argparse
import json
import pathlib
import statistics
import sys
from typing import Any, Callable

import torch

REPO_ROOT = pathlib.Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))

from rl_engine.mhc.fp32_gemm_rms import fp32_gemm_rms # noqa: E402

N_DIM = 24
EPS = 1e-6


def torch_native(x: torch.Tensor, w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
p = x @ w.t()
r = 1.0 / (torch.sqrt(x.square().mean(dim=1)) + EPS)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

bug · high
The torch-native reference derives r from the input x, not from the GEMM projection p = x @ w.t(). If the operated fp32_gemm_rms defines r as the row-wise RMS of the projection (the fused gemm+rms this backend is named for), then this reference is a different workload: r is independent of w, so in forward_backward the dr upstream gradient never contributes to w.grad. The forward/backward timing then measures a different graph than the strict Triton/CUDA backends it is contrasted with, making the comparison non-equivalent. Please align the derivation (and eps placement) with the operator's actual contract before drawing conclusions from these numbers.

Suggestion:

Suggested change
r = 1.0 / (torch.sqrt(x.square().mean(dim=1)) + EPS)
r = 1.0 / (torch.sqrt(p.square().mean(dim=1)) + EPS)

return p, r


def _runner(backend: str) -> Callable[[torch.Tensor, torch.Tensor], tuple]:
if backend == "torch-native":
return torch_native
return lambda x, w: fp32_gemm_rms(x, w, EPS, backend=backend)


def _time_ms(fn: Callable[[], None], warmup: int, iterations: int) -> dict[str, float]:
for _ in range(warmup):
fn()
torch.cuda.synchronize()
samples = []
for _ in range(iterations):
start = torch.cuda.Event(enable_timing=True)
stop = torch.cuda.Event(enable_timing=True)
start.record()
fn()
stop.record()
torch.cuda.synchronize()
samples.append(start.elapsed_time(stop))
return {
"median_ms": statistics.median(samples),
"mean_ms": statistics.fmean(samples),
"min_ms": min(samples),
"max_ms": max(samples),
"samples": samples,
}


def bench_case(backend: str, tokens: int, k: int, warmup: int, iterations: int) -> dict[str, Any]:
gen = torch.Generator(device="cpu").manual_seed(tokens)
x0 = torch.randn(tokens, k, generator=gen, dtype=torch.float32).cuda()
w0 = torch.randn(N_DIM, k, generator=gen, dtype=torch.float32).cuda()
dp = torch.randn(tokens, N_DIM, generator=gen, dtype=torch.float32).cuda()
dr = torch.randn(tokens, generator=gen, dtype=torch.float32).cuda()
run = _runner(backend)

def forward_only() -> None:
with torch.no_grad():
run(x0, w0)

def forward_backward() -> None:
x = x0.detach().requires_grad_(True)
w = w0.detach().requires_grad_(True)
p, r = run(x, w)
torch.autograd.backward([p, r], [dp, dr])

return {
"forward": _time_ms(forward_only, warmup, iterations),
"forward_backward": _time_ms(forward_backward, warmup, iterations),
}


def main() -> int:
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument("--tokens", default="1,16,128,512")
parser.add_argument("--k", type=int, default=16384)
parser.add_argument("--warmup", type=int, default=5)
parser.add_argument("--iterations", type=int, default=20)
parser.add_argument("--backends", default="torch-native,triton,cuda")
parser.add_argument("--json", dest="json_path", default=None)
args = parser.parse_args()

if not torch.cuda.is_available():
print("CUDA is required for this benchmark", file=sys.stderr)
return 1

token_list = [int(v) for v in args.tokens.split(",")]
backends = args.backends.split(",")
report: dict[str, Any] = {
"device": torch.cuda.get_device_name(),
"torch": torch.__version__,
"k": args.k,
"n": N_DIM,
"warmup": args.warmup,
"iterations": args.iterations,
"results": {},
}

header = f"{'T':>6} {'mode':<18}" + "".join(f"{b:>16}" for b in backends)
print(header)
print("-" * len(header))
for tokens in token_list:
rows: dict[str, Any] = {}
for backend in backends:
try:
rows[backend] = bench_case(backend, tokens, args.k, args.warmup, args.iterations)
except (RuntimeError, ValueError, TypeError) as exc:
rows[backend] = {"error": str(exc)}
report["results"][str(tokens)] = rows
for mode in ("forward", "forward_backward"):
cells = []
for backend in backends:
entry = rows[backend]
if "error" in entry:
cells.append(f"{'ERROR':>16}")
else:
cells.append(f"{entry[mode]['median_ms']:>15.6f} ")
print(f"{tokens:>6} {mode:<18}" + "".join(cells))

if args.json_path:
with open(args.json_path, "w") as fh:
json.dump(report, fh, indent=2)
print(f"\nreport written to {args.json_path}")
return 0


if __name__ == "__main__":
sys.exit(main())
90 changes: 90 additions & 0 deletions benchmarks/benchmark_fused_shared_expert_mlp.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# Copyright (c) 2026 RL-Kernel Contributors
"""Fused shared-expert fc1 + SwiGLU vs the unfused det_gemm composition.

Both paths are byte-identical (tests/test_fused_shared_expert_mlp.py asserts
it), so this measures only what fusing buys. The BF16 cuBLAS column is the
non-deterministic speed ceiling, not an alternative: it is neither
batch-invariant nor TP-equivalent.

python benchmarks/benchmark_fused_shared_expert_mlp.py [--tokens 512,2048,8192]
"""

from __future__ import annotations

import argparse
import pathlib
import statistics
import sys

import torch

REPO_ROOT = pathlib.Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))


def time_ms(fn, warmup: int = 10, iters: int = 50, repeats: int = 5) -> float:
"""Median of `repeats` timed runs -- the first samples are unstable."""
for _ in range(warmup):
fn()
samples = []
for _ in range(repeats):
torch.cuda.synchronize()
start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
start.record()
for _ in range(iters):
fn()
end.record()
torch.cuda.synchronize()
samples.append(start.elapsed_time(end) / iters)
return statistics.median(samples)


def main() -> int:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--hidden", type=int, default=4096)
p.add_argument("--ffn", type=int, default=2048)
p.add_argument("--tokens", default="128,512,2048,8192,32768")
args = p.parse_args()
if not torch.cuda.is_available():
print("CUDA device required")
return 1
try:
from rl_engine import _C
except ImportError as exc:
print(f"[skip] rl_engine._C is not built: {exc}")
return 1
for sym in ("fused_shared_expert_fc1_swiglu", "det_gemm_fwd_rhs_transposed"):
if not hasattr(_C, sym):
print(f"[skip] rl_engine._C lacks {sym}; rebuild the extension")
return 1

H, F = args.hidden, args.ffn
print(f"H={H} F={F} ({torch.cuda.get_device_name(0)})")
print(f"{'tokens':>7} {'unfused':>10} {'fused':>9} {'speedup':>8} {'bf16 cuBLAS':>13}")
for T in [int(v) for v in args.tokens.split(",")]:
gen = torch.Generator().manual_seed(1)
x = (torch.randn(T, H, generator=gen) * 0.7).to(torch.bfloat16).cuda()
w1 = (torch.randn(2 * F, H, generator=gen) / H**0.5).to(torch.bfloat16).cuda()

def unfused():
z = _C.det_gemm_fwd_rhs_transposed(x, w1).float()
gate, up = z[:, :F], z[:, F:]
return ((gate * torch.sigmoid(gate)) * up).to(torch.bfloat16)

def fused():
return _C.fused_shared_expert_fc1_swiglu(x, w1)

def cublas():
z = x @ w1.t()
return (torch.nn.functional.silu(z[:, :F]) * z[:, F:]).to(torch.bfloat16)

t_un, t_fu, t_cb = time_ms(unfused), time_ms(fused), time_ms(cublas)
print(f"{T:>7} {t_un:10.3f} {t_fu:9.3f} {t_un / t_fu:7.2f}x {t_cb:13.3f}")
return 0


if __name__ == "__main__":
sys.exit(main())
Loading
Loading