From 495a48f7a1e9f520ee88175810ad0ec001a5f34d Mon Sep 17 00:00:00 2001 From: mmelnich Date: Thu, 29 Oct 2026 12:13:58 -0700 Subject: [PATCH] Add a coverage build option and CI job RandLAPACK had no coverage measurement. This adds one, as a signal rather than a gate. - RandLAPACK_ENABLE_COVERAGE (GCC only): test/CMakeLists.txt instruments the C++ test translation units with --coverage at -O0, which is where a header-only library's code is compiled, and warns if the build type is not Debug. - test/misc/test_instantiations.cc explicitly instantiates, for float and double and in headers no open PR touches, class templates and the member and function templates the tests miss (BK's sparse-matrix call, the sparse-matrix and sketching-operator applies of SparseLinOp and CompositeOperator, standardize_dataset). An uninstantiated template contributes no lines to gcov, so it was invisible rather than uncovered. test/drivers/test_instantiations_gpu.cu does the same for the two GPU drivers in the CUDA test executable. - The coverage workflow builds and runs the core suite that way, reports with gcovr (HTML artifact, summary on the job page, Cobertura XML) and uploads to Codecov through OpenID Connect with informational statuses (codecov.yml), so nothing is blocked on a percentage. Lines are merged across template instantiations; functions are counted per instantiation, and branches merge only where the instantiations' control flow matches, so untested float versions can lower those two numbers. - docs/CI.md (the job, what its numbers count, a local recipe), CONTRIBUTING.md and docs/INSTALL_MANUAL.md describe the option. Verified locally (GCC 15.2, oneMKL ILP64, RTX 5070 Ti), on 18ddcedb: - Debug build with the option, reported with the local recipe in docs/CI.md: 531 of 533 tests passed, 2 skipped (thread-cap tests); lines 4484/5258 (85.3%), branches 3413/6878, functions 521/728. - Release build with CUDA: 587 of 589 tests passed (CPU and GPU, ctest -LE exclusive), the same 2 skipped; every float instantiation compiles. - On c9706937, before this commit the report read lines 4326/5058, branches 3232/6109, functions 470/552; with it, lines 4326/5094. Fixes #127. --- .github/workflows/coverage.yaml | 138 ++++++++++++++++++++++++ CMake/rl_build_options.cmake | 1 + CONTRIBUTING.md | 8 ++ codecov.yml | 20 ++++ docs/CI.md | 49 ++++++++- docs/INSTALL_MANUAL.md | 1 + test/CMakeLists.txt | 16 +++ test/drivers/test_instantiations_gpu.cu | 11 ++ test/misc/test_instantiations.cc | 52 +++++++++ 9 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/coverage.yaml create mode 100644 codecov.yml create mode 100644 test/drivers/test_instantiations_gpu.cu create mode 100644 test/misc/test_instantiations.cc diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml new file mode 100644 index 00000000..4d90a8a4 --- /dev/null +++ b/.github/workflows/coverage.yaml @@ -0,0 +1,138 @@ +name: coverage +on: + pull_request: + branches-ignore: + - cqrrp-gpu-benchmarking + workflow_dispatch: + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +# Coverage of the library by the core test suite, built with RandLAPACK_ENABLE_COVERAGE (GCC, Debug). +# Reported, never enforced (codecov.yml); docs/CI.md explains the numbers. The Codecov upload uses +# OpenID Connect, needs the Codecov GitHub App on the repository, and never fails the job. + +jobs: + coverage: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: submodule update + run: | + git submodule update --init --recursive + + - name: configure OS + run: | + set -x + export DEBIAN_FRONTEND="noninteractive" + sudo apt-get update -qq + sudo apt-get install -qq -y git-core gcc g++ gfortran cmake libgtest-dev libopenblas-openmp-dev + pipx install gcovr==8.8 + + - name: read dependency pins + id: pins + run: bash .github/scripts/dependency-pins.sh export installers/dependency-pins.env + + - name: cache dependencies + id: deps-cache + uses: actions/cache@v4 + with: + path: | + ${{ runner.workspace }}/blaspp-install + ${{ runner.workspace }}/lapackpp-install + ${{ runner.workspace }}/Random123-install + key: core-deps-${{ runner.os }}-blaspp-${{ steps.pins.outputs.BLASPP_REF }}-lapackpp-${{ steps.pins.outputs.LAPACKPP_REF }}-v1 + + - name: install BLAS++ + if: steps.deps-cache.outputs.cache-hit != 'true' + env: + BLASPP_URL: ${{ steps.pins.outputs.BLASPP_URL }} + BLASPP_REF: ${{ steps.pins.outputs.BLASPP_REF }} + run: | + cd .. + bash "$GITHUB_WORKSPACE/.github/scripts/dependency-pins.sh" fetch "$BLASPP_URL" "$BLASPP_REF" blaspp + mkdir blaspp-build + cd blaspp-build + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=`pwd`/../blaspp-install -Dbuild_tests=OFF ../blaspp + make -j$(nproc) install + + - name: install Random123 + if: steps.deps-cache.outputs.cache-hit != 'true' + run: | + cd .. + git clone https://github.com/DEShawResearch/Random123.git + cd Random123/ + make prefix=`pwd`/../Random123-install install-include + + - name: install LAPACK++ + if: steps.deps-cache.outputs.cache-hit != 'true' + env: + LAPACKPP_URL: ${{ steps.pins.outputs.LAPACKPP_URL }} + LAPACKPP_REF: ${{ steps.pins.outputs.LAPACKPP_REF }} + run: | + cd .. + bash "$GITHUB_WORKSPACE/.github/scripts/dependency-pins.sh" fetch "$LAPACKPP_URL" "$LAPACKPP_REF" lapackpp + mkdir lapackpp-build + cd lapackpp-build + cmake -DCMAKE_BUILD_TYPE=Release \ + -Dblaspp_DIR=`pwd`/../blaspp-install/lib/cmake/blaspp \ + -DCMAKE_INSTALL_PREFIX=`pwd`/../lapackpp-install \ + -Dbuild_tests=OFF \ + `pwd`/../lapackpp + make -j$(nproc) install + + - name: build and test RandLAPACK (Debug, coverage) + env: + OMP_NUM_THREADS: 2 + run: | + cd .. + mkdir RandLAPACK-build-cov + cd RandLAPACK-build-cov + cmake -DCMAKE_BUILD_TYPE=Debug \ + -DRandLAPACK_ENABLE_COVERAGE=ON \ + -Dblaspp_DIR=`pwd`/../blaspp-install/lib/cmake/blaspp \ + -Dlapackpp_DIR=`pwd`/../lapackpp-install/lib/cmake/lapackpp \ + -DRandom123_DIR=`pwd`/../Random123-install/include/ \ + -DBUILD_TESTS=OFF \ + `pwd`/../RandLAPACK + make -j$(nproc) + ctest --output-on-failure -j "$(nproc)" + + - name: coverage report + run: | + cd ../RandLAPACK-build-cov + mkdir -p coverage + gcovr --root "$GITHUB_WORKSPACE" \ + --filter "$GITHUB_WORKSPACE/RandLAPACK/" \ + --exclude-throw-branches \ + --exclude-unreachable-branches \ + --merge-lines \ + -j "$(nproc)" \ + --html-details coverage/index.html \ + --cobertura coverage/coverage.xml \ + --markdown-summary coverage/summary.md \ + --txt coverage/coverage.txt \ + . + cat coverage/summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: upload the HTML report + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: ${{ runner.workspace }}/RandLAPACK-build-cov/coverage/ + + - name: upload to Codecov + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + with: + use_oidc: true + files: ${{ runner.workspace }}/RandLAPACK-build-cov/coverage/coverage.xml + disable_search: true + fail_ci_if_error: false diff --git a/CMake/rl_build_options.cmake b/CMake/rl_build_options.cmake index 4f80eae6..87b6567a 100644 --- a/CMake/rl_build_options.cmake +++ b/CMake/rl_build_options.cmake @@ -10,6 +10,7 @@ if(NOT CMAKE_BUILD_TYPE) endif() set(SANITIZE_ADDRESS OFF CACHE BOOL "Add address sanitizer flags to the library") +option(RandLAPACK_ENABLE_COVERAGE "Instrument the tests for gcov line and branch coverage (GCC only)" OFF) message(STATUS "Checking for OpenMP ... ") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f505e49..199f57be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,6 +87,14 @@ and removes it afterwards, never a fixed or `rand()`-named path. `ScratchDir` in `extras/test/ext_scratch_dir.hh` does this for the extras tests, and the fixture in `test/misc/test_matrix_io.cc` for the core suite. +**Coverage is measured, not required.** The `coverage` workflow builds the +suite with `-DRandLAPACK_ENABLE_COVERAGE=ON` (GCC, Debug) and reports which +library lines and branches the tests execute; read its report for the lines +your PR adds. Template code that nothing instantiates is invisible to it +rather than counted as uncovered, so a new template that no test instantiates goes into +`test/misc/test_instantiations.cc` ([docs/CI.md](docs/CI.md#coverage) has the +details and the local command). + **GPU tests are not run by CI.** CI neither compiles nor runs them (`test/**/*.cu`; [#215](https://github.com/BallisticLA/RandLAPACK/issues/215)). If your change touches GPU code, build and run them on a machine with CUDA, diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..256a2c9b --- /dev/null +++ b/codecov.yml @@ -0,0 +1,20 @@ +# Coverage is reported, never enforced: both statuses are informational, so they cannot block a merge. +# The coverage workflow already restricts the report to the library headers; the ignores below keep a +# report generated some other way to the same scope. +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true + +comment: + layout: "condensed_header, condensed_files, condensed_footer" + +ignore: + - "test/**" + - "benchmark/**" + - "extras/**" + - "RandBLAS/**" diff --git a/docs/CI.md b/docs/CI.md index 41656333..1136a540 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -1,6 +1,6 @@ # RandLAPACK's GitHub CI -Five workflows run on every push to `main` and on every pull request, +Six workflows run on every push to `main` and on every pull request, except pull requests into the `cqrrp-gpu-benchmarking` branch: - **core-linux, core-macos, core-windows** replicate the build recipe by @@ -8,6 +8,8 @@ except pull requests into the `cqrrp-gpu-benchmarking` branch: - **install-script** runs the user-facing installers end to end, so the commands we tell users to type are themselves under test. - **dependency-pins** checks the BLAS++/LAPACK++ pins that every lane builds. +- **coverage** measures which lines and branches of the library the core + test suite executes. Each operating system is thus covered by two workflows, a core one and the installer one. @@ -21,6 +23,7 @@ installer one. | core-windows | `windows-toolchain-guards` | windows-2022 | Architecture-guard decision table, plus assertions that `install.ps1` *refuses* a real x86 and a cross-compiled arm64 toolchain, launched exactly as the docs prescribe | no | | core-windows | `build-windows` | windows-2022 | MSVC build + tests: oneMKL ILP64 serial, oneMKL ILP64 OpenMP (`/openmp:llvm`), OpenBLAS LP64 serial ("serial" = RandLAPACK's own loops; the BLAS backend is multithreaded in every leg); each leg ends with a stripped-PATH run of a staged test executable | no | | core-windows | `windows-spaced-path` | windows-2022 | oneMKL ILP64 copied under `Program Files (x86)\Intel\oneAPI\mkl\latest`, once through `-Backend mkl -MklRoot` and once through `-Backend custom`: fresh BLAS++/LAPACK++ builds whose exported config must name that spaced path, then the test suite | no | +| coverage | `coverage` | ubuntu-latest | Debug build with `RandLAPACK_ENABLE_COVERAGE=ON` (GCC, OpenBLAS), test run, gcovr report: HTML artifact `coverage-report`, summary on the job page, upload to Codecov | no | | dependency-pins | `pins` | ubuntu-latest | `installers/dependency-pins.env`: each commit exists at its URL, a non-empty tag points at its commit, and the pins match the RandBLAS submodule's | no | | install-script | `install-linux` | ubuntu-latest | `install.sh`: fresh install, idempotent re-run, dependency-discovery path | **yes** | | install-script | `install-linux-openblas64` | ubuntu-latest | `install.sh --blas=openblas` with Debian/Ubuntu's `libopenblas64` as the only OpenBLAS: BLAS++ must come out ILP64 and link `libopenblas64`, then the test suite | no | @@ -46,6 +49,17 @@ installer one. - **Linux/macOS core recipe:** follow the steps in the workflow file; they are ordinary `cmake` + `make` invocations. +- **Coverage**, with GCC and `pipx install gcovr==8.8`, from the repository + root (add the dependency paths your build needs to the first command): + + ```bash + cmake -S . -B coverage-build -DCMAKE_BUILD_TYPE=Debug -DRandLAPACK_ENABLE_COVERAGE=ON -DBUILD_TESTS=OFF + cmake --build coverage-build -j 4 && (cd coverage-build && OMP_NUM_THREADS=2 ctest -j 4) + mkdir -p coverage-build/coverage + gcovr --root . --filter RandLAPACK/ --exclude-throw-branches \ + --exclude-unreachable-branches --merge-lines \ + --html-details coverage-build/coverage/index.html coverage-build + ``` - **Installer lanes:** `bash install.sh --yes --no-gpu` from a fresh clone. - **Windows**, from an x64 developer prompt in the repository root: `powershell -ExecutionPolicy Bypass -File .github\scripts\windows\run-ci.ps1 -Task Core -SetupDependencies` @@ -90,8 +104,8 @@ installer one. process, so test cases that share a file or a directory fail here, as they do under a local `ctest -j`; serial runs never caught that. `OMP_NUM_THREADS` is set low on those steps so that the parallel tests do - not oversubscribe the runner. The other jobs still run their tests - serially. + not oversubscribe the runner. The coverage job runs its tests the same way; + the other jobs still run theirs serially. - **The RandBLAS submodule's own tests do not run here** (`-DBUILD_TESTS=OFF` in the core recipes). RandBLAS's CI already tests the pinned commit, and rebuilding its roughly 480 tests in every RandLAPACK job @@ -103,6 +117,33 @@ installer one. CUDA translation units, so a change that breaks GPU code still passes CI. Build and run the GPU tests locally when you touch GPU code. +### Coverage + +- **Coverage is reported, not enforced.** The job is not a required check, + and the Codecov project and patch statuses are informational + (`codecov.yml`). A percentage says which lines ran, not whether their + results were checked; read the patch report in review instead of chasing + the total. +- **GCC, Debug, `-O0`.** gcov's line counts follow the source only without + optimization, and the option uses GCC's gcov flags. +- **The library is header-only templates, so code counts only once + something instantiates it.** An uninstantiated template produces no lines + at all; it is invisible, not 0%. `test/misc/test_instantiations.cc` + therefore instantiates class templates, and member and function templates + that the tests miss, for `float` and `double`. When you add a template + that no test instantiates, add it there. +- **What the numbers count.** Lines are merged across template + instantiations (`--merge-lines`): a line counts as covered if any + instantiation ran it. Functions are counted per instantiation, and + branches merge only where the instantiations' control flow matches, so a + `float` version that no test runs can lower both. +- **GPU code is not in the report.** It compiles only in CUDA translation + units, which this job does not build. +- **The Codecov upload uses OpenID Connect, not a stored token.** It needs + the Codecov GitHub App installed for the repository; until then the upload + step logs an error and passes, and the HTML artifact and the job summary + still carry the report. + ### Windows - **The Windows matrix is deliberately small** (CI-cost budget). The two mkl @@ -228,7 +269,7 @@ a cache-miss run is several minutes slower than a warm one. | Cache key prefix | Used by | Lives beside | |------------------|---------|--------------| -| `core-deps--v` | core-linux, core-macos (both jobs) | `runner.workspace`, the checkout's parent (`*-install`) | +| `core-deps--v` | core-linux, core-macos (both jobs), coverage (shares core-linux's key) | `runner.workspace`, the checkout's parent (`*-install`) | | `installer-deps--v` | install-linux | `deps-install/` in the workspace | | `windows-*-r` (per backend + shared) | core-windows (`build-windows`, `windows-spaced-path`), install-windows | `runner.workspace\windows-deps` | diff --git a/docs/INSTALL_MANUAL.md b/docs/INSTALL_MANUAL.md index f3549d04..9373f37b 100644 --- a/docs/INSTALL_MANUAL.md +++ b/docs/INSTALL_MANUAL.md @@ -174,6 +174,7 @@ Other optional flags: |---|---| | `-DRandLAPACK_BUILD_TESTS=OFF` | Skips RandLAPACK's own GoogleTest suite, which is built by default when GoogleTest is found. | | `-DSANITIZE_ADDRESS=ON` | Adds AddressSanitizer instrumentation (CI uses it with Debug builds). | +| `-DRandLAPACK_ENABLE_COVERAGE=ON` | Instruments the test suite for gcov line and branch coverage (GCC only; forces `-O0`; see [CI.md](CI.md#coverage)). | The `extras/` project has a third test switch of its own, `-DBUILD_TESTING=OFF`. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8365bcfc..97669175 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,6 +1,20 @@ message(STATUS "Checking for regression tests ...") find_package(GTest) +# The library is header-only, so its code is compiled in the test translation units: instrumenting +# them measures it. -O0 keeps gcov's line counts faithful to the source. CUDA sources are not instrumented. +if (RandLAPACK_ENABLE_COVERAGE) + if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + message(FATAL_ERROR "RandLAPACK_ENABLE_COVERAGE=ON requires GCC, not ${CMAKE_CXX_COMPILER_ID}.") + endif() + if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + message(WARNING "RandLAPACK_ENABLE_COVERAGE=ON is meant for Debug builds, " + "not CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}.") + endif() + add_compile_options("$<$:--coverage;-O0;-g;-fprofile-abs-path;-fprofile-update=prefer-atomic>") + add_link_options("$<$:--coverage>") +endif() + set(tmp FALSE) if (GTest_FOUND) set(tmp TRUE) @@ -41,6 +55,7 @@ if (GTest_FOUND) misc/test_matrix_io.cc misc/test_memory_tracker.cc linops/test_linop_block_views.cc + misc/test_instantiations.cc ) # Create non-CUDA test executable @@ -64,6 +79,7 @@ if (GTest_FOUND) comps/test_util_gpu.cu drivers/test_cqrrpt_gpu.cu drivers/test_bqrrp_gpu.cu + drivers/test_instantiations_gpu.cu ) # Create CUDA test executables diff --git a/test/drivers/test_instantiations_gpu.cu b/test/drivers/test_instantiations_gpu.cu new file mode 100644 index 00000000..df8b63d7 --- /dev/null +++ b/test/drivers/test_instantiations_gpu.cu @@ -0,0 +1,11 @@ +// Explicit instantiations of the GPU drivers for float and double, the counterpart of +// test/misc/test_instantiations.cc for the CUDA test executable. + +#include "RandLAPACK.hh" + +using RNG = RandBLAS::DefaultRNG; + +template class RandLAPACK::BQRRP_GPU; +template class RandLAPACK::BQRRP_GPU; +template class RandLAPACK::CQRRPT_GPU; +template class RandLAPACK::CQRRPT_GPU; diff --git a/test/misc/test_instantiations.cc b/test/misc/test_instantiations.cc new file mode 100644 index 00000000..347baa28 --- /dev/null +++ b/test/misc/test_instantiations.cc @@ -0,0 +1,52 @@ +// Explicit float and double instantiations: a template contributes no lines to a coverage report +// until something instantiates it, so this file covers class templates and member and function +// templates that the tests miss. It also compiles the float versions no test builds. + +#include "RandLAPACK.hh" + +using RNG = RandBLAS::DefaultRNG; +template using CSR = RandBLAS::sparse_data::CSRMatrix; +template using DenseComposite = + RandLAPACK::linops::CompositeOperator, RandLAPACK::linops::DenseLinOp>; + +// operator() overloads whose other operand is a sparse matrix or a sketching operator. +#define RL_INSTANTIATE_APPLY(T, Op_t, B_t) \ + template void Op_t::operator()(Layout, blas::Op, blas::Op, int64_t, int64_t, int64_t, \ + T, B_t&, T, T*, int64_t); + +#define RL_INSTANTIATE(T) \ + template class RandLAPACK::CholQRQ; \ + template class RandLAPACK::HQRQ; \ + template class RandLAPACK::PLUL; \ + template class RandLAPACK::RS; \ + template class RandLAPACK::RF; \ + template class RandLAPACK::QB; \ + template class RandLAPACK::RSVD; \ + template class RandLAPACK::SYPS; \ + template class RandLAPACK::SYRF, RandLAPACK::HQRQ>; \ + template class RandLAPACK::BK; \ + template class RandLAPACK::ABRIK; \ + template class RandLAPACK::BQRRP; \ + template class RandLAPACK::CQRRPT; \ + template struct RandLAPACK::StatefulFrobeniusNorm; \ + template struct RandLAPACK::linops::RBFKernelMatrix; \ + template struct RandLAPACK::linops::CSRRowBlockView; \ + template struct RandLAPACK::linops::CSRColBlock; \ + template struct RandLAPACK::linops::CSCColBlockView; \ + template struct RandLAPACK::linops::CSCRowBlock; \ + template struct RandLAPACK::linops::SparseLinOp>; \ + template struct RandLAPACK::linops::SparseLinOp>; \ + template class RandLAPACK::linops::DowndatableLinOp>; \ + template struct RandLAPACK::linops::CompositeOperator< \ + RandLAPACK::linops::DenseLinOp, RandLAPACK::linops::DenseLinOp>; \ + template int RandLAPACK::BK::call>( \ + int64_t, int64_t, CSR&, int64_t, int64_t, \ + T*&, T*&, T*&, T*&, int64_t&, int64_t&, bool&, RandBLAS::RNGState&); \ + RL_INSTANTIATE_APPLY(T, RandLAPACK::linops::SparseLinOp>, CSR) \ + RL_INSTANTIATE_APPLY(T, RandLAPACK::linops::SparseLinOp>, RandBLAS::DenseSkOp) \ + RL_INSTANTIATE_APPLY(T, DenseComposite, CSR) \ + RL_INSTANTIATE_APPLY(T, DenseComposite, RandBLAS::DenseSkOp) \ + template void RandLAPACK::standardize_dataset(int64_t, int64_t, T*, T*, T*, bool); + +RL_INSTANTIATE(float) +RL_INSTANTIATE(double)