diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml new file mode 100644 index 00000000..4d90a8a4 --- /dev/null +++ b/.github/workflows/coverage.yaml @@ -0,0 +1,138 @@ +name: coverage +on: + pull_request: + branches-ignore: + - cqrrp-gpu-benchmarking + workflow_dispatch: + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +# Coverage of the library by the core test suite, built with RandLAPACK_ENABLE_COVERAGE (GCC, Debug). +# Reported, never enforced (codecov.yml); docs/CI.md explains the numbers. The Codecov upload uses +# OpenID Connect, needs the Codecov GitHub App on the repository, and never fails the job. + +jobs: + coverage: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: submodule update + run: | + git submodule update --init --recursive + + - name: configure OS + run: | + set -x + export DEBIAN_FRONTEND="noninteractive" + sudo apt-get update -qq + sudo apt-get install -qq -y git-core gcc g++ gfortran cmake libgtest-dev libopenblas-openmp-dev + pipx install gcovr==8.8 + + - name: read dependency pins + id: pins + run: bash .github/scripts/dependency-pins.sh export installers/dependency-pins.env + + - name: cache dependencies + id: deps-cache + uses: actions/cache@v4 + with: + path: | + ${{ runner.workspace }}/blaspp-install + ${{ runner.workspace }}/lapackpp-install + ${{ runner.workspace }}/Random123-install + key: core-deps-${{ runner.os }}-blaspp-${{ steps.pins.outputs.BLASPP_REF }}-lapackpp-${{ steps.pins.outputs.LAPACKPP_REF }}-v1 + + - name: install BLAS++ + if: steps.deps-cache.outputs.cache-hit != 'true' + env: + BLASPP_URL: ${{ steps.pins.outputs.BLASPP_URL }} + BLASPP_REF: ${{ steps.pins.outputs.BLASPP_REF }} + run: | + cd .. + bash "$GITHUB_WORKSPACE/.github/scripts/dependency-pins.sh" fetch "$BLASPP_URL" "$BLASPP_REF" blaspp + mkdir blaspp-build + cd blaspp-build + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=`pwd`/../blaspp-install -Dbuild_tests=OFF ../blaspp + make -j$(nproc) install + + - name: install Random123 + if: steps.deps-cache.outputs.cache-hit != 'true' + run: | + cd .. + git clone https://github.com/DEShawResearch/Random123.git + cd Random123/ + make prefix=`pwd`/../Random123-install install-include + + - name: install LAPACK++ + if: steps.deps-cache.outputs.cache-hit != 'true' + env: + LAPACKPP_URL: ${{ steps.pins.outputs.LAPACKPP_URL }} + LAPACKPP_REF: ${{ steps.pins.outputs.LAPACKPP_REF }} + run: | + cd .. + bash "$GITHUB_WORKSPACE/.github/scripts/dependency-pins.sh" fetch "$LAPACKPP_URL" "$LAPACKPP_REF" lapackpp + mkdir lapackpp-build + cd lapackpp-build + cmake -DCMAKE_BUILD_TYPE=Release \ + -Dblaspp_DIR=`pwd`/../blaspp-install/lib/cmake/blaspp \ + -DCMAKE_INSTALL_PREFIX=`pwd`/../lapackpp-install \ + -Dbuild_tests=OFF \ + `pwd`/../lapackpp + make -j$(nproc) install + + - name: build and test RandLAPACK (Debug, coverage) + env: + OMP_NUM_THREADS: 2 + run: | + cd .. + mkdir RandLAPACK-build-cov + cd RandLAPACK-build-cov + cmake -DCMAKE_BUILD_TYPE=Debug \ + -DRandLAPACK_ENABLE_COVERAGE=ON \ + -Dblaspp_DIR=`pwd`/../blaspp-install/lib/cmake/blaspp \ + -Dlapackpp_DIR=`pwd`/../lapackpp-install/lib/cmake/lapackpp \ + -DRandom123_DIR=`pwd`/../Random123-install/include/ \ + -DBUILD_TESTS=OFF \ + `pwd`/../RandLAPACK + make -j$(nproc) + ctest --output-on-failure -j "$(nproc)" + + - name: coverage report + run: | + cd ../RandLAPACK-build-cov + mkdir -p coverage + gcovr --root "$GITHUB_WORKSPACE" \ + --filter "$GITHUB_WORKSPACE/RandLAPACK/" \ + --exclude-throw-branches \ + --exclude-unreachable-branches \ + --merge-lines \ + -j "$(nproc)" \ + --html-details coverage/index.html \ + --cobertura coverage/coverage.xml \ + --markdown-summary coverage/summary.md \ + --txt coverage/coverage.txt \ + . + cat coverage/summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: upload the HTML report + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: ${{ runner.workspace }}/RandLAPACK-build-cov/coverage/ + + - name: upload to Codecov + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + with: + use_oidc: true + files: ${{ runner.workspace }}/RandLAPACK-build-cov/coverage/coverage.xml + disable_search: true + fail_ci_if_error: false diff --git a/CMake/rl_build_options.cmake b/CMake/rl_build_options.cmake index 4f80eae6..87b6567a 100644 --- a/CMake/rl_build_options.cmake +++ b/CMake/rl_build_options.cmake @@ -10,6 +10,7 @@ if(NOT CMAKE_BUILD_TYPE) endif() set(SANITIZE_ADDRESS OFF CACHE BOOL "Add address sanitizer flags to the library") +option(RandLAPACK_ENABLE_COVERAGE "Instrument the tests for gcov line and branch coverage (GCC only)" OFF) message(STATUS "Checking for OpenMP ... ") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f505e49..199f57be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,6 +87,14 @@ and removes it afterwards, never a fixed or `rand()`-named path. `ScratchDir` in `extras/test/ext_scratch_dir.hh` does this for the extras tests, and the fixture in `test/misc/test_matrix_io.cc` for the core suite. +**Coverage is measured, not required.** The `coverage` workflow builds the +suite with `-DRandLAPACK_ENABLE_COVERAGE=ON` (GCC, Debug) and reports which +library lines and branches the tests execute; read its report for the lines +your PR adds. Template code that nothing instantiates is invisible to it +rather than counted as uncovered, so a new template that no test instantiates goes into +`test/misc/test_instantiations.cc` ([docs/CI.md](docs/CI.md#coverage) has the +details and the local command). + **GPU tests are not run by CI.** CI neither compiles nor runs them (`test/**/*.cu`; [#215](https://github.com/BallisticLA/RandLAPACK/issues/215)). If your change touches GPU code, build and run them on a machine with CUDA, diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..256a2c9b --- /dev/null +++ b/codecov.yml @@ -0,0 +1,20 @@ +# Coverage is reported, never enforced: both statuses are informational, so they cannot block a merge. +# The coverage workflow already restricts the report to the library headers; the ignores below keep a +# report generated some other way to the same scope. +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true + +comment: + layout: "condensed_header, condensed_files, condensed_footer" + +ignore: + - "test/**" + - "benchmark/**" + - "extras/**" + - "RandBLAS/**" diff --git a/docs/CI.md b/docs/CI.md index 2918cb1b..9c034865 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -1,6 +1,6 @@ # RandLAPACK's GitHub CI -Five workflows run on every push to `main` and on every pull request, +Six workflows run on every push to `main` and on every pull request, except pull requests into the `cqrrp-gpu-benchmarking` branch: - **core-linux, core-macos, core-windows** replicate the build recipe by @@ -8,6 +8,8 @@ except pull requests into the `cqrrp-gpu-benchmarking` branch: - **install-script** runs the user-facing installers end to end, so the commands we tell users to type are themselves under test. - **dependency-pins** checks the BLAS++/LAPACK++ pins that every lane builds. +- **coverage** measures which lines and branches of the library the core + test suite executes. Each operating system is thus covered by two workflows, a core one and the installer one. @@ -21,6 +23,7 @@ installer one. | core-windows | `windows-toolchain-guards` | windows-2022 | Architecture-guard decision table, plus assertions that `install.ps1` *refuses* a real x86 and a cross-compiled arm64 toolchain, launched exactly as the docs prescribe | no | | core-windows | `build-windows` | windows-2022 | MSVC build + tests: oneMKL ILP64 serial, oneMKL ILP64 OpenMP (`/openmp:llvm`), OpenBLAS LP64 serial ("serial" = RandLAPACK's own loops; the BLAS backend is multithreaded in every leg); each leg ends with a stripped-PATH run of a staged test executable | no | | core-windows | `windows-spaced-path` | windows-2022 | oneMKL ILP64 copied under `Program Files (x86)\Intel\oneAPI\mkl\latest`, once through `-Backend mkl -MklRoot` and once through `-Backend custom`: fresh BLAS++/LAPACK++ builds whose exported config must name that spaced path, then the test suite | no | +| coverage | `coverage` | ubuntu-latest | Debug build with `RandLAPACK_ENABLE_COVERAGE=ON` (GCC, OpenBLAS), test run, gcovr report: HTML artifact `coverage-report`, summary on the job page, upload to Codecov | no | | dependency-pins | `pins` | ubuntu-latest | `installers/dependency-pins.env`: each commit exists at its URL, a non-empty tag points at its commit, and the pins match the RandBLAS submodule's | no | | install-script | `install-linux` | ubuntu-latest | `install.sh`: fresh install, idempotent re-run, dependency-discovery path | **yes** | | install-script | `install-linux-openblas64` | ubuntu-latest | `install.sh --blas=openblas` with Debian/Ubuntu's `libopenblas64` as the only OpenBLAS: BLAS++ must come out ILP64 and link `libopenblas64`, then the test suite | no | @@ -46,6 +49,17 @@ installer one. - **Linux/macOS core recipe:** follow the steps in the workflow file; they are ordinary `cmake` + `make` invocations. +- **Coverage**, with GCC and `pipx install gcovr==8.8`, from the repository + root (add the dependency paths your build needs to the first command): + + ```bash + cmake -S . -B coverage-build -DCMAKE_BUILD_TYPE=Debug -DRandLAPACK_ENABLE_COVERAGE=ON -DBUILD_TESTS=OFF + cmake --build coverage-build -j 4 && (cd coverage-build && OMP_NUM_THREADS=2 ctest -j 4) + mkdir -p coverage-build/coverage + gcovr --root . --filter RandLAPACK/ --exclude-throw-branches \ + --exclude-unreachable-branches --merge-lines \ + --html-details coverage-build/coverage/index.html coverage-build + ``` - **Installer lanes:** `bash install.sh --yes --no-gpu` from a fresh clone. - **Windows**, from an x64 developer prompt in the repository root: `powershell -ExecutionPolicy Bypass -File .github\scripts\windows\run-ci.ps1 -Task Core -SetupDependencies` @@ -90,8 +104,8 @@ installer one. process, so test cases that share a file or a directory fail here, as they do under a local `ctest -j`; serial runs never caught that. `OMP_NUM_THREADS` is set low on those steps so that the parallel tests do - not oversubscribe the runner. The other jobs still run their tests - serially. + not oversubscribe the runner. The coverage job runs its tests the same way; + the other jobs still run theirs serially. - **The RandBLAS submodule's own tests do not run here** (`-DBUILD_TESTS=OFF` in the core recipes). RandBLAS's CI already tests the pinned commit, and rebuilding its roughly 480 tests in every RandLAPACK job @@ -103,6 +117,33 @@ installer one. CUDA translation units, so a change that breaks GPU code still passes CI. Build and run the GPU tests locally when you touch GPU code. +### Coverage + +- **Coverage is reported, not enforced.** The job is not a required check, + and the Codecov project and patch statuses are informational + (`codecov.yml`). A percentage says which lines ran, not whether their + results were checked; read the patch report in review instead of chasing + the total. +- **GCC, Debug, `-O0`.** gcov's line counts follow the source only without + optimization, and the option uses GCC's gcov flags. +- **The library is header-only templates, so code counts only once + something instantiates it.** An uninstantiated template produces no lines + at all; it is invisible, not 0%. `test/misc/test_instantiations.cc` + therefore instantiates class templates, and member and function templates + that the tests miss, for `float` and `double`. When you add a template + that no test instantiates, add it there. +- **What the numbers count.** Lines are merged across template + instantiations (`--merge-lines`): a line counts as covered if any + instantiation ran it. Functions are counted per instantiation, and + branches merge only where the instantiations' control flow matches, so a + `float` version that no test runs can lower both. +- **GPU code is not in the report.** It compiles only in CUDA translation + units, which this job does not build. +- **The Codecov upload uses OpenID Connect, not a stored token.** It needs + the Codecov GitHub App installed for the repository; until then the upload + step logs an error and passes, and the HTML artifact and the job summary + still carry the report. + ### Windows - **The Windows matrix is deliberately small** (CI-cost budget). The two mkl @@ -230,7 +271,7 @@ a cache-miss run is several minutes slower than a warm one. | Cache key prefix | Used by | Lives beside | |------------------|---------|--------------| -| `core-deps--v` | core-linux, core-macos (both jobs) | `runner.workspace`, the checkout's parent (`*-install`) | +| `core-deps--v` | core-linux, core-macos (both jobs), coverage (shares core-linux's key) | `runner.workspace`, the checkout's parent (`*-install`) | | `installer-deps--v` | install-linux | `deps-install/` in the workspace | | `windows-*-r` (per backend + shared) | core-windows (`build-windows`, `windows-spaced-path`), install-windows | `runner.workspace\windows-deps` | diff --git a/docs/INSTALL_MANUAL.md b/docs/INSTALL_MANUAL.md index f3549d04..9373f37b 100644 --- a/docs/INSTALL_MANUAL.md +++ b/docs/INSTALL_MANUAL.md @@ -174,6 +174,7 @@ Other optional flags: |---|---| | `-DRandLAPACK_BUILD_TESTS=OFF` | Skips RandLAPACK's own GoogleTest suite, which is built by default when GoogleTest is found. | | `-DSANITIZE_ADDRESS=ON` | Adds AddressSanitizer instrumentation (CI uses it with Debug builds). | +| `-DRandLAPACK_ENABLE_COVERAGE=ON` | Instruments the test suite for gcov line and branch coverage (GCC only; forces `-O0`; see [CI.md](CI.md#coverage)). | The `extras/` project has a third test switch of its own, `-DBUILD_TESTING=OFF`. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dc631fc9..a7b7c10d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,6 +1,20 @@ message(STATUS "Checking for regression tests ...") find_package(GTest) +# The library is header-only, so its code is compiled in the test translation units: instrumenting +# them measures it. -O0 keeps gcov's line counts faithful to the source. CUDA sources are not instrumented. +if (RandLAPACK_ENABLE_COVERAGE) + if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + message(FATAL_ERROR "RandLAPACK_ENABLE_COVERAGE=ON requires GCC, not ${CMAKE_CXX_COMPILER_ID}.") + endif() + if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + message(WARNING "RandLAPACK_ENABLE_COVERAGE=ON is meant for Debug builds, " + "not CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}.") + endif() + add_compile_options("$<$:--coverage;-O0;-g;-fprofile-abs-path;-fprofile-update=prefer-atomic>") + add_link_options("$<$:--coverage>") +endif() + set(tmp FALSE) if (GTest_FOUND) set(tmp TRUE) @@ -46,6 +60,7 @@ if (GTest_FOUND) misc/test_memory_tracker.cc misc/test_bench_common.cc linops/test_linop_block_views.cc + misc/test_instantiations.cc ) # Benchmark helpers tested here live in benchmark/, outside the library. @@ -73,6 +88,7 @@ if (GTest_FOUND) comps/test_util_gpu.cu drivers/test_cqrrpt_gpu.cu drivers/test_bqrrp_gpu.cu + drivers/test_instantiations_gpu.cu ) # Create CUDA test executables diff --git a/test/drivers/test_instantiations_gpu.cu b/test/drivers/test_instantiations_gpu.cu new file mode 100644 index 00000000..df8b63d7 --- /dev/null +++ b/test/drivers/test_instantiations_gpu.cu @@ -0,0 +1,11 @@ +// Explicit instantiations of the GPU drivers for float and double, the counterpart of +// test/misc/test_instantiations.cc for the CUDA test executable. + +#include "RandLAPACK.hh" + +using RNG = RandBLAS::DefaultRNG; + +template class RandLAPACK::BQRRP_GPU; +template class RandLAPACK::BQRRP_GPU; +template class RandLAPACK::CQRRPT_GPU; +template class RandLAPACK::CQRRPT_GPU; diff --git a/test/misc/test_instantiations.cc b/test/misc/test_instantiations.cc new file mode 100644 index 00000000..347baa28 --- /dev/null +++ b/test/misc/test_instantiations.cc @@ -0,0 +1,52 @@ +// Explicit float and double instantiations: a template contributes no lines to a coverage report +// until something instantiates it, so this file covers class templates and member and function +// templates that the tests miss. It also compiles the float versions no test builds. + +#include "RandLAPACK.hh" + +using RNG = RandBLAS::DefaultRNG; +template using CSR = RandBLAS::sparse_data::CSRMatrix; +template using DenseComposite = + RandLAPACK::linops::CompositeOperator, RandLAPACK::linops::DenseLinOp>; + +// operator() overloads whose other operand is a sparse matrix or a sketching operator. +#define RL_INSTANTIATE_APPLY(T, Op_t, B_t) \ + template void Op_t::operator()(Layout, blas::Op, blas::Op, int64_t, int64_t, int64_t, \ + T, B_t&, T, T*, int64_t); + +#define RL_INSTANTIATE(T) \ + template class RandLAPACK::CholQRQ; \ + template class RandLAPACK::HQRQ; \ + template class RandLAPACK::PLUL; \ + template class RandLAPACK::RS; \ + template class RandLAPACK::RF; \ + template class RandLAPACK::QB; \ + template class RandLAPACK::RSVD; \ + template class RandLAPACK::SYPS; \ + template class RandLAPACK::SYRF, RandLAPACK::HQRQ>; \ + template class RandLAPACK::BK; \ + template class RandLAPACK::ABRIK; \ + template class RandLAPACK::BQRRP; \ + template class RandLAPACK::CQRRPT; \ + template struct RandLAPACK::StatefulFrobeniusNorm; \ + template struct RandLAPACK::linops::RBFKernelMatrix; \ + template struct RandLAPACK::linops::CSRRowBlockView; \ + template struct RandLAPACK::linops::CSRColBlock; \ + template struct RandLAPACK::linops::CSCColBlockView; \ + template struct RandLAPACK::linops::CSCRowBlock; \ + template struct RandLAPACK::linops::SparseLinOp>; \ + template struct RandLAPACK::linops::SparseLinOp>; \ + template class RandLAPACK::linops::DowndatableLinOp>; \ + template struct RandLAPACK::linops::CompositeOperator< \ + RandLAPACK::linops::DenseLinOp, RandLAPACK::linops::DenseLinOp>; \ + template int RandLAPACK::BK::call>( \ + int64_t, int64_t, CSR&, int64_t, int64_t, \ + T*&, T*&, T*&, T*&, int64_t&, int64_t&, bool&, RandBLAS::RNGState&); \ + RL_INSTANTIATE_APPLY(T, RandLAPACK::linops::SparseLinOp>, CSR) \ + RL_INSTANTIATE_APPLY(T, RandLAPACK::linops::SparseLinOp>, RandBLAS::DenseSkOp) \ + RL_INSTANTIATE_APPLY(T, DenseComposite, CSR) \ + RL_INSTANTIATE_APPLY(T, DenseComposite, RandBLAS::DenseSkOp) \ + template void RandLAPACK::standardize_dataset(int64_t, int64_t, T*, T*, T*, bool); + +RL_INSTANTIATE(float) +RL_INSTANTIATE(double)