Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 138 additions & 0 deletions .github/workflows/coverage.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
name: coverage
on:
pull_request:
branches-ignore:
- cqrrp-gpu-benchmarking
workflow_dispatch:
push:
branches:
- main

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

# Coverage of the library by the core test suite, built with RandLAPACK_ENABLE_COVERAGE (GCC, Debug).
# Reported, never enforced (codecov.yml); docs/CI.md explains the numbers. The Codecov upload uses
# OpenID Connect, needs the Codecov GitHub App on the repository, and never fails the job.

jobs:
coverage:
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@v4

- name: submodule update
run: |
git submodule update --init --recursive

- name: configure OS
run: |
set -x
export DEBIAN_FRONTEND="noninteractive"
sudo apt-get update -qq
sudo apt-get install -qq -y git-core gcc g++ gfortran cmake libgtest-dev libopenblas-openmp-dev
pipx install gcovr==8.8

- name: read dependency pins
id: pins
run: bash .github/scripts/dependency-pins.sh export installers/dependency-pins.env

- name: cache dependencies
id: deps-cache
uses: actions/cache@v4
with:
path: |
${{ runner.workspace }}/blaspp-install
${{ runner.workspace }}/lapackpp-install
${{ runner.workspace }}/Random123-install
key: core-deps-${{ runner.os }}-blaspp-${{ steps.pins.outputs.BLASPP_REF }}-lapackpp-${{ steps.pins.outputs.LAPACKPP_REF }}-v1

- name: install BLAS++
if: steps.deps-cache.outputs.cache-hit != 'true'
env:
BLASPP_URL: ${{ steps.pins.outputs.BLASPP_URL }}
BLASPP_REF: ${{ steps.pins.outputs.BLASPP_REF }}
run: |
cd ..
bash "$GITHUB_WORKSPACE/.github/scripts/dependency-pins.sh" fetch "$BLASPP_URL" "$BLASPP_REF" blaspp
mkdir blaspp-build
cd blaspp-build
cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=`pwd`/../blaspp-install -Dbuild_tests=OFF ../blaspp
make -j$(nproc) install

- name: install Random123
if: steps.deps-cache.outputs.cache-hit != 'true'
run: |
cd ..
git clone https://github.com/DEShawResearch/Random123.git
cd Random123/
make prefix=`pwd`/../Random123-install install-include

- name: install LAPACK++
if: steps.deps-cache.outputs.cache-hit != 'true'
env:
LAPACKPP_URL: ${{ steps.pins.outputs.LAPACKPP_URL }}
LAPACKPP_REF: ${{ steps.pins.outputs.LAPACKPP_REF }}
run: |
cd ..
bash "$GITHUB_WORKSPACE/.github/scripts/dependency-pins.sh" fetch "$LAPACKPP_URL" "$LAPACKPP_REF" lapackpp
mkdir lapackpp-build
cd lapackpp-build
cmake -DCMAKE_BUILD_TYPE=Release \
-Dblaspp_DIR=`pwd`/../blaspp-install/lib/cmake/blaspp \
-DCMAKE_INSTALL_PREFIX=`pwd`/../lapackpp-install \
-Dbuild_tests=OFF \
`pwd`/../lapackpp
make -j$(nproc) install

- name: build and test RandLAPACK (Debug, coverage)
env:
OMP_NUM_THREADS: 2
run: |
cd ..
mkdir RandLAPACK-build-cov
cd RandLAPACK-build-cov
cmake -DCMAKE_BUILD_TYPE=Debug \
-DRandLAPACK_ENABLE_COVERAGE=ON \
-Dblaspp_DIR=`pwd`/../blaspp-install/lib/cmake/blaspp \
-Dlapackpp_DIR=`pwd`/../lapackpp-install/lib/cmake/lapackpp \
-DRandom123_DIR=`pwd`/../Random123-install/include/ \
-DBUILD_TESTS=OFF \
`pwd`/../RandLAPACK
make -j$(nproc)
ctest --output-on-failure -j "$(nproc)"

- name: coverage report
run: |
cd ../RandLAPACK-build-cov
mkdir -p coverage
gcovr --root "$GITHUB_WORKSPACE" \
--filter "$GITHUB_WORKSPACE/RandLAPACK/" \
--exclude-throw-branches \
--exclude-unreachable-branches \
--merge-lines \
-j "$(nproc)" \
--html-details coverage/index.html \
--cobertura coverage/coverage.xml \
--markdown-summary coverage/summary.md \
--txt coverage/coverage.txt \
.
cat coverage/summary.md >> "$GITHUB_STEP_SUMMARY"

- name: upload the HTML report
uses: actions/upload-artifact@v4
with:
name: coverage-report
path: ${{ runner.workspace }}/RandLAPACK-build-cov/coverage/

- name: upload to Codecov
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
with:
use_oidc: true
files: ${{ runner.workspace }}/RandLAPACK-build-cov/coverage/coverage.xml
disable_search: true
fail_ci_if_error: false
1 change: 1 addition & 0 deletions CMake/rl_build_options.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ if(NOT CMAKE_BUILD_TYPE)
endif()

set(SANITIZE_ADDRESS OFF CACHE BOOL "Add address sanitizer flags to the library")
option(RandLAPACK_ENABLE_COVERAGE "Instrument the tests for gcov line and branch coverage (GCC only)" OFF)

message(STATUS "Checking for OpenMP ... ")

Expand Down
8 changes: 8 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,14 @@ and removes it afterwards, never a fixed or `rand()`-named path. `ScratchDir`
in `extras/test/ext_scratch_dir.hh` does this for the extras tests, and the
fixture in `test/misc/test_matrix_io.cc` for the core suite.

**Coverage is measured, not required.** The `coverage` workflow builds the
suite with `-DRandLAPACK_ENABLE_COVERAGE=ON` (GCC, Debug) and reports which
library lines and branches the tests execute; read its report for the lines
your PR adds. Template code that nothing instantiates is invisible to it
rather than counted as uncovered, so a new template that no test instantiates goes into
`test/misc/test_instantiations.cc` ([docs/CI.md](docs/CI.md#coverage) has the
details and the local command).

**GPU tests are not run by CI.** CI neither compiles nor runs them
(`test/**/*.cu`; [#215](https://github.com/BallisticLA/RandLAPACK/issues/215)).
If your change touches GPU code, build and run them on a machine with CUDA,
Expand Down
20 changes: 20 additions & 0 deletions codecov.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Coverage is reported, never enforced: both statuses are informational, so they cannot block a merge.
# The coverage workflow already restricts the report to the library headers; the ignores below keep a
# report generated some other way to the same scope.
coverage:
status:
project:
default:
informational: true
patch:
default:
informational: true

comment:
layout: "condensed_header, condensed_files, condensed_footer"

ignore:
- "test/**"
- "benchmark/**"
- "extras/**"
- "RandBLAS/**"
49 changes: 45 additions & 4 deletions docs/CI.md
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
# RandLAPACK's GitHub CI

Five workflows run on every push to `main` and on every pull request,
Six workflows run on every push to `main` and on every pull request,
except pull requests into the `cqrrp-gpu-benchmarking` branch:

- **core-linux, core-macos, core-windows** replicate the build recipe by
hand, step by step, so a failure points at a specific step.
- **install-script** runs the user-facing installers end to end, so the
commands we tell users to type are themselves under test.
- **dependency-pins** checks the BLAS++/LAPACK++ pins that every lane builds.
- **coverage** measures which lines and branches of the library the core
test suite executes.

Each operating system is thus covered by two workflows, a core one and the
installer one.
Expand All @@ -21,6 +23,7 @@ installer one.
| core-windows | `windows-toolchain-guards` | windows-2022 | Architecture-guard decision table, plus assertions that `install.ps1` *refuses* a real x86 and a cross-compiled arm64 toolchain, launched exactly as the docs prescribe | no |
| core-windows | `build-windows` | windows-2022 | MSVC build + tests: oneMKL ILP64 serial, oneMKL ILP64 OpenMP (`/openmp:llvm`), OpenBLAS LP64 serial ("serial" = RandLAPACK's own loops; the BLAS backend is multithreaded in every leg); each leg ends with a stripped-PATH run of a staged test executable | no |
| core-windows | `windows-spaced-path` | windows-2022 | oneMKL ILP64 copied under `Program Files (x86)\Intel\oneAPI\mkl\latest`, once through `-Backend mkl -MklRoot` and once through `-Backend custom`: fresh BLAS++/LAPACK++ builds whose exported config must name that spaced path, then the test suite | no |
| coverage | `coverage` | ubuntu-latest | Debug build with `RandLAPACK_ENABLE_COVERAGE=ON` (GCC, OpenBLAS), test run, gcovr report: HTML artifact `coverage-report`, summary on the job page, upload to Codecov | no |
| dependency-pins | `pins` | ubuntu-latest | `installers/dependency-pins.env`: each commit exists at its URL, a non-empty tag points at its commit, and the pins match the RandBLAS submodule's | no |
| install-script | `install-linux` | ubuntu-latest | `install.sh`: fresh install, idempotent re-run, dependency-discovery path | **yes** |
| install-script | `install-linux-openblas64` | ubuntu-latest | `install.sh --blas=openblas` with Debian/Ubuntu's `libopenblas64` as the only OpenBLAS: BLAS++ must come out ILP64 and link `libopenblas64`, then the test suite | no |
Expand All @@ -46,6 +49,17 @@ installer one.

- **Linux/macOS core recipe:** follow the steps in the workflow file; they
are ordinary `cmake` + `make` invocations.
- **Coverage**, with GCC and `pipx install gcovr==8.8`, from the repository
root (add the dependency paths your build needs to the first command):

```bash
cmake -S . -B coverage-build -DCMAKE_BUILD_TYPE=Debug -DRandLAPACK_ENABLE_COVERAGE=ON -DBUILD_TESTS=OFF
cmake --build coverage-build -j 4 && (cd coverage-build && OMP_NUM_THREADS=2 ctest -j 4)
mkdir -p coverage-build/coverage
gcovr --root . --filter RandLAPACK/ --exclude-throw-branches \
--exclude-unreachable-branches --merge-lines \
--html-details coverage-build/coverage/index.html coverage-build
```
- **Installer lanes:** `bash install.sh --yes --no-gpu` from a fresh clone.
- **Windows**, from an x64 developer prompt in the repository root:
`powershell -ExecutionPolicy Bypass -File .github\scripts\windows\run-ci.ps1 -Task Core -SetupDependencies`
Expand Down Expand Up @@ -90,8 +104,8 @@ installer one.
process, so test cases that share a file or a directory fail here, as they
do under a local `ctest -j`; serial runs never caught that.
`OMP_NUM_THREADS` is set low on those steps so that the parallel tests do
not oversubscribe the runner. The other jobs still run their tests
serially.
not oversubscribe the runner. The coverage job runs its tests the same way;
the other jobs still run theirs serially.
- **The RandBLAS submodule's own tests do not run here**
(`-DBUILD_TESTS=OFF` in the core recipes). RandBLAS's CI already tests the
pinned commit, and rebuilding its roughly 480 tests in every RandLAPACK job
Expand All @@ -103,6 +117,33 @@ installer one.
CUDA translation units, so a change that breaks GPU code still passes CI.
Build and run the GPU tests locally when you touch GPU code.

### Coverage

- **Coverage is reported, not enforced.** The job is not a required check,
and the Codecov project and patch statuses are informational
(`codecov.yml`). A percentage says which lines ran, not whether their
results were checked; read the patch report in review instead of chasing
the total.
- **GCC, Debug, `-O0`.** gcov's line counts follow the source only without
optimization, and the option uses GCC's gcov flags.
- **The library is header-only templates, so code counts only once
something instantiates it.** An uninstantiated template produces no lines
at all; it is invisible, not 0%. `test/misc/test_instantiations.cc`
therefore instantiates class templates, and member and function templates
that the tests miss, for `float` and `double`. When you add a template
that no test instantiates, add it there.
- **What the numbers count.** Lines are merged across template
instantiations (`--merge-lines`): a line counts as covered if any
instantiation ran it. Functions are counted per instantiation, and
branches merge only where the instantiations' control flow matches, so a
`float` version that no test runs can lower both.
- **GPU code is not in the report.** It compiles only in CUDA translation
units, which this job does not build.
- **The Codecov upload uses OpenID Connect, not a stored token.** It needs
the Codecov GitHub App installed for the repository; until then the upload
step logs an error and passes, and the HTML artifact and the job summary
still carry the report.

### Windows

- **The Windows matrix is deliberately small** (CI-cost budget). The two mkl
Expand Down Expand Up @@ -230,7 +271,7 @@ a cache-miss run is several minutes slower than a warm one.

| Cache key prefix | Used by | Lives beside |
|------------------|---------|--------------|
| `core-deps-<OS>-v<N>` | core-linux, core-macos (both jobs) | `runner.workspace`, the checkout's parent (`*-install`) |
| `core-deps-<OS>-v<N>` | core-linux, core-macos (both jobs), coverage (shares core-linux's key) | `runner.workspace`, the checkout's parent (`*-install`) |
| `installer-deps-<OS>-v<N>` | install-linux | `deps-install/` in the workspace |
| `windows-*-r<N>` (per backend + shared) | core-windows (`build-windows`, `windows-spaced-path`), install-windows | `runner.workspace\windows-deps` |

Expand Down
1 change: 1 addition & 0 deletions docs/INSTALL_MANUAL.md
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ Other optional flags:
|---|---|
| `-DRandLAPACK_BUILD_TESTS=OFF` | Skips RandLAPACK's own GoogleTest suite, which is built by default when GoogleTest is found. |
| `-DSANITIZE_ADDRESS=ON` | Adds AddressSanitizer instrumentation (CI uses it with Debug builds). |
| `-DRandLAPACK_ENABLE_COVERAGE=ON` | Instruments the test suite for gcov line and branch coverage (GCC only; forces `-O0`; see [CI.md](CI.md#coverage)). |

The `extras/` project has a third test switch of its own,
`-DBUILD_TESTING=OFF`.
Expand Down
16 changes: 16 additions & 0 deletions test/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,20 @@
message(STATUS "Checking for regression tests ...")
find_package(GTest)

# The library is header-only, so its code is compiled in the test translation units: instrumenting
# them measures it. -O0 keeps gcov's line counts faithful to the source. CUDA sources are not instrumented.
if (RandLAPACK_ENABLE_COVERAGE)
if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
message(FATAL_ERROR "RandLAPACK_ENABLE_COVERAGE=ON requires GCC, not ${CMAKE_CXX_COMPILER_ID}.")
endif()
if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
message(WARNING "RandLAPACK_ENABLE_COVERAGE=ON is meant for Debug builds, "
"not CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}.")
endif()
add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:--coverage;-O0;-g;-fprofile-abs-path;-fprofile-update=prefer-atomic>")
add_link_options("$<$<LINK_LANGUAGE:CXX>:--coverage>")
endif()

set(tmp FALSE)
if (GTest_FOUND)
set(tmp TRUE)
Expand Down Expand Up @@ -46,6 +60,7 @@ if (GTest_FOUND)
misc/test_memory_tracker.cc
misc/test_bench_common.cc
linops/test_linop_block_views.cc
misc/test_instantiations.cc
)

# Benchmark helpers tested here live in benchmark/, outside the library.
Expand Down Expand Up @@ -73,6 +88,7 @@ if (GTest_FOUND)
comps/test_util_gpu.cu
drivers/test_cqrrpt_gpu.cu
drivers/test_bqrrp_gpu.cu
drivers/test_instantiations_gpu.cu
)

# Create CUDA test executables
Expand Down
11 changes: 11 additions & 0 deletions test/drivers/test_instantiations_gpu.cu
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
// Explicit instantiations of the GPU drivers for float and double, the counterpart of
// test/misc/test_instantiations.cc for the CUDA test executable.

#include "RandLAPACK.hh"

using RNG = RandBLAS::DefaultRNG;

template class RandLAPACK::BQRRP_GPU<float, RNG>;
template class RandLAPACK::BQRRP_GPU<double, RNG>;
template class RandLAPACK::CQRRPT_GPU<float, RNG>;
template class RandLAPACK::CQRRPT_GPU<double, RNG>;
52 changes: 52 additions & 0 deletions test/misc/test_instantiations.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
// Explicit float and double instantiations: a template contributes no lines to a coverage report
// until something instantiates it, so this file covers class templates and member and function
// templates that the tests miss. It also compiles the float versions no test builds.

#include "RandLAPACK.hh"

using RNG = RandBLAS::DefaultRNG;
template <typename T> using CSR = RandBLAS::sparse_data::CSRMatrix<T>;
template <typename T> using DenseComposite =
RandLAPACK::linops::CompositeOperator<RandLAPACK::linops::DenseLinOp<T>, RandLAPACK::linops::DenseLinOp<T>>;

// operator() overloads whose other operand is a sparse matrix or a sketching operator.
#define RL_INSTANTIATE_APPLY(T, Op_t, B_t) \
template void Op_t::operator()<B_t>(Layout, blas::Op, blas::Op, int64_t, int64_t, int64_t, \
T, B_t&, T, T*, int64_t);

#define RL_INSTANTIATE(T) \
template class RandLAPACK::CholQRQ<T>; \
template class RandLAPACK::HQRQ<T>; \
template class RandLAPACK::PLUL<T>; \
template class RandLAPACK::RS<T, RNG>; \
template class RandLAPACK::RF<T, RNG>; \
template class RandLAPACK::QB<T, RNG>; \
template class RandLAPACK::RSVD<T, RNG>; \
template class RandLAPACK::SYPS<T, RNG>; \
template class RandLAPACK::SYRF<RandLAPACK::SYPS<T, RNG>, RandLAPACK::HQRQ<T>>; \
template class RandLAPACK::BK<T, RNG>; \
template class RandLAPACK::ABRIK<T, RNG>; \
template class RandLAPACK::BQRRP<T, RNG>; \
template class RandLAPACK::CQRRPT<T, RNG>; \
template struct RandLAPACK::StatefulFrobeniusNorm<T>; \
template struct RandLAPACK::linops::RBFKernelMatrix<T>; \
template struct RandLAPACK::linops::CSRRowBlockView<T>; \
template struct RandLAPACK::linops::CSRColBlock<T>; \
template struct RandLAPACK::linops::CSCColBlockView<T>; \
template struct RandLAPACK::linops::CSCRowBlock<T>; \
template struct RandLAPACK::linops::SparseLinOp<CSR<T>>; \
template struct RandLAPACK::linops::SparseLinOp<RandBLAS::sparse_data::CSCMatrix<T>>; \
template class RandLAPACK::linops::DowndatableLinOp<T, RandLAPACK::linops::DenseLinOp<T>>; \
template struct RandLAPACK::linops::CompositeOperator< \
RandLAPACK::linops::DenseLinOp<T>, RandLAPACK::linops::DenseLinOp<T>>; \
template int RandLAPACK::BK<T, RNG>::call<CSR<T>>( \
int64_t, int64_t, CSR<T>&, int64_t, int64_t, \
T*&, T*&, T*&, T*&, int64_t&, int64_t&, bool&, RandBLAS::RNGState<RNG>&); \
RL_INSTANTIATE_APPLY(T, RandLAPACK::linops::SparseLinOp<CSR<T>>, CSR<T>) \
RL_INSTANTIATE_APPLY(T, RandLAPACK::linops::SparseLinOp<CSR<T>>, RandBLAS::DenseSkOp<T>) \
RL_INSTANTIATE_APPLY(T, DenseComposite<T>, CSR<T>) \
RL_INSTANTIATE_APPLY(T, DenseComposite<T>, RandBLAS::DenseSkOp<T>) \
template void RandLAPACK::standardize_dataset<T>(int64_t, int64_t, T*, T*, T*, bool);

RL_INSTANTIATE(float)
RL_INSTANTIATE(double)
Loading