diff --git a/docs/01_user_guide.md b/docs/01_user_guide.md index c532a1a0..547e08aa 100644 --- a/docs/01_user_guide.md +++ b/docs/01_user_guide.md @@ -71,6 +71,7 @@ The import process uses the global **Mass Tolerance** setting (Default: 0.2 Da) * To check or change this: Go to **Settings → Extraction**, **Mass tolerance (Da)**, before or after loading data. * If CDFs are already loaded, changing the tolerance regenerates every EIC. You do not re-import the folder. Each EIC keeps its tR window, including one set with **Apply window**. +* If a sample's CDF file is missing or unreadable at that point, the sample keeps its old-tolerance EICs. Its plot captions say **old mass tolerance**, export asks you to confirm before it writes the file, and its cells in the exported workbook are light gold. Put the file back and change the tolerance again to rebuild the sample and clear the mark. **Procedure** @@ -335,6 +336,7 @@ The Excel file contains five worksheets by default. A sixth sheet is optional. **Validation & Errors** * **Invalid Peaks:** Cells corresponding to peaks that failed the minimum area validation (plots with a **Below threshold** badge) will be highlighted with a **light red background** in the Excel file. +* **Old mass tolerance:** Every cell of a sample whose EICs were built at an earlier mass tolerance has a **light gold background**. This mark replaces any validation colour for that sample. * **Abundance Errors:** If the internal standard is missing required calibration fields (`int_std_amount` or `amount_in_std_mix`), MANIC will prompt you to either **cancel** and fix the compound list, or **continue** in unnormalized **Peak Area** mode for that export. --- diff --git a/docs/Reference_Mass_Tolerance.md b/docs/Reference_Mass_Tolerance.md index 7a199534..e357c569 100644 --- a/docs/Reference_Mass_Tolerance.md +++ b/docs/Reference_Mass_Tolerance.md @@ -50,6 +50,7 @@ A mass that lies exactly on a window edge depends on how the instrument stored i * **Range:** `0.01` to `1.00` Da * **Impact:** This setting is applied when EICs are extracted. * If CDFs are already loaded, changing the value regenerates every EIC. Each EIC keeps its tR window. You do not re-import the folder. + * A sample whose CDF file is missing or unreadable keeps its EICs from the old tolerance. Its plot captions say **old mass tolerance** and its cells in the exported workbook are light gold until a later tolerance change rebuilds it. * Plots and integrations refresh after that regeneration finishes. --- diff --git a/src/manic/io/changelog_sections.py b/src/manic/io/changelog_sections.py index d4b51b42..a6e51d34 100644 --- a/src/manic/io/changelog_sections.py +++ b/src/manic/io/changelog_sections.py @@ -223,6 +223,7 @@ def format_cell_colour_key() -> str: f"- Light red ({PEAK_VERDICT_FILL[PeakVerdict.FAIL]}) = below internal-standard threshold\n" f"- Light purple ({PEAK_VERDICT_FILL[PeakVerdict.ACCEPTED]}) = accepted by the analyst\n" f"- Tan ({PEAK_VERDICT_FILL[PeakVerdict.REJECTED]}) = marked bad by the analyst\n" + f"- Light gold ({PEAK_VERDICT_FILL[PeakVerdict.STALE]}) = whole sample, EICs from an older mass tolerance\n" f"- Blue compound name ({BASELINE_OFF_FONT_COLOR}) = baseline correction off\n" ) diff --git a/src/manic/io/data_exporter.py b/src/manic/io/data_exporter.py index a24e44a4..16d2b079 100644 --- a/src/manic/io/data_exporter.py +++ b/src/manic/io/data_exporter.py @@ -22,6 +22,7 @@ from manic.constants import DEFAULT_MASS_TOLERANCE, DEFAULT_MIN_PEAK_HEIGHT_RATIO from manic.io.changelog_writer import generate_changelog from manic.io.data_provider import DataProvider +from manic.io.sample_reader import list_stale_samples from manic.models.analysis import AnalysisMode from manic.models.peak_review import get_peak_reviews from manic.validation.peak_verdict import PeakVerdict @@ -186,11 +187,17 @@ def _compute_validation_data( validation_disabled = ( not self.internal_standard_compound or self.min_peak_area_ratio <= 0 ) - if validation_disabled and not reviews: - return {} + stale = set(list_stale_samples()) validation_data = {} for sample in samples: + if sample in stale: + validation_data[sample] = { + compound["compound_name"]: PeakVerdict.STALE for compound in compounds + } + continue + if validation_disabled and not reviews: + continue validation_data[sample] = {} for compound in compounds: compound_name = compound["compound_name"] diff --git a/src/manic/io/eic_importer.py b/src/manic/io/eic_importer.py index 90f03085..6080025f 100644 --- a/src/manic/io/eic_importer.py +++ b/src/manic/io/eic_importer.py @@ -920,6 +920,11 @@ def regenerate_all_eics_with_mass_tolerance( regenerated = 0 failed_samples = [] + # Each sample clears its flag in the transaction that replaces its EICs, so + # a reload that fails or stops partway leaves the rest marked. + with get_connection() as conn: + conn.execute("UPDATE samples SET eics_stale = 1") + # Process each sample file for sample_row in sample_rows: sample_name = sample_row["sample_name"] @@ -1038,6 +1043,10 @@ def regenerate_all_eics_with_mass_tolerance( logger.debug( f"Inserted {len(eic_batch)} new EIC records for '{sample_name}'" ) + conn.execute( + "UPDATE samples SET eics_stale = 0 WHERE sample_name = ?", + (sample_name,), + ) regenerated += len(eic_batch) done += len(compounds) diff --git a/src/manic/io/sample_reader.py b/src/manic/io/sample_reader.py index f6e5cad9..152f9a74 100644 --- a/src/manic/io/sample_reader.py +++ b/src/manic/io/sample_reader.py @@ -30,3 +30,12 @@ def list_active_samples() -> List[str]: # `conn.row_factory` is sqlite3.Row, so each row works like a dict return [row["sample_name"] for row in rows] + + +def list_stale_samples() -> List[str]: + """Samples, deleted ones included, whose EICs missed the last mass tolerance change.""" + with get_connection() as conn: + rows = conn.execute( + "SELECT sample_name FROM samples WHERE eics_stale = 1 ORDER BY sample_name" + ).fetchall() + return [row["sample_name"] for row in rows] diff --git a/src/manic/models/database.py b/src/manic/models/database.py index 24ef6414..5edcd089 100644 --- a/src/manic/models/database.py +++ b/src/manic/models/database.py @@ -215,6 +215,12 @@ def _run_migrations(conn: sqlite3.Connection) -> None: logger.error(f"Migration error for compounds table: {e}") pass + columns = [row[1] for row in conn.execute("PRAGMA table_info(samples)")] + if columns and "eics_stale" not in columns: + logger.info("Adding eics_stale column to samples table") + conn.execute("ALTER TABLE samples ADD COLUMN eics_stale INTEGER DEFAULT 0") + conn.commit() + @contextmanager diff --git a/src/manic/models/schema.sql b/src/manic/models/schema.sql index c4e73078..b79f4f5e 100644 --- a/src/manic/models/schema.sql +++ b/src/manic/models/schema.sql @@ -50,6 +50,7 @@ CREATE TABLE IF NOT EXISTS samples ( id INTEGER PRIMARY KEY AUTOINCREMENT, sample_name TEXT NOT NULL UNIQUE, file_name TEXT, + eics_stale INTEGER DEFAULT 0, deleted INTEGER DEFAULT 0 ); diff --git a/src/manic/ui/graphs.py b/src/manic/ui/graphs.py index 6aa51349..ea4a72f9 100644 --- a/src/manic/ui/graphs.py +++ b/src/manic/ui/graphs.py @@ -24,6 +24,7 @@ ) from manic.io.compound_reader import read_compound_with_session +from manic.io.sample_reader import list_stale_samples from manic.models.sample_fit_type import FIT_TYPE_LABELS, get_sample_fit_types from manic.processors.eic_processing import get_eics_for_compound from manic.processors.display_deconvolution import ( @@ -97,10 +98,13 @@ def retention_time_text(values: Iterable[float]) -> str: return f"tR {_minutes(low)}–{_minutes(high)}" -def tile_caption(sample_name: str, fit_type: str | None) -> str: - if not fit_type: - return sample_name - return f"{sample_name} · {FIT_TYPE_LABELS.get(fit_type, fit_type)}" +def tile_caption(sample_name: str, fit_type: str | None, stale: bool = False) -> str: + parts = [sample_name] + if fit_type: + parts.append(FIT_TYPE_LABELS.get(fit_type, fit_type)) + if stale: + parts.append("old mass tolerance") + return " · ".join(parts) def qc_key_html(identity: IdentityAssessmentSet | None) -> str: @@ -370,6 +374,7 @@ def __init__(self, parent=None): self._current_compound: str = "" self._current_samples: List[str] = [] self._sample_fit_types: Dict[tuple[str, str], str] = {} + self._stale_samples: Set[str] = set() self.use_corrected = False self._prepared_displays = {} @@ -573,6 +578,7 @@ def plot_compound( self._current_compound = compound_name self._current_samples = samples self._sample_fit_types = get_sample_fit_types(compound_name) + self._stale_samples = set(list_stale_samples()) with measure_time("get_eics_from_db"): eics = get_eics_for_compound( @@ -1402,7 +1408,9 @@ def _create_plot_container( def _tile_caption_for(self, eic) -> str: fit_type = self._sample_fit_types.get((eic.compound_name, eic.sample_name)) - return tile_caption(eic.sample_name, fit_type) + return tile_caption( + eic.sample_name, fit_type, eic.sample_name in self._stale_samples + ) def _update_container_data( self, diff --git a/src/manic/ui/main_window.py b/src/manic/ui/main_window.py index 85aa8f99..ba257571 100644 --- a/src/manic/ui/main_window.py +++ b/src/manic/ui/main_window.py @@ -56,7 +56,7 @@ from manic.io.data_provider import DataProvider from manic.io.eic_importer import reextract_session_eics from manic.io.list_compound_names import list_compound_names -from manic.io.sample_reader import list_active_samples +from manic.io.sample_reader import list_active_samples, list_stale_samples from manic.io.compound_reader import ( compound_reads_cached, list_compound_mm_files, @@ -2989,6 +2989,19 @@ def export_data(self): if options.exec() != QDialog.Accepted: return + active = set(list_active_samples()) + stale = [name for name in list_stale_samples() if name in active] + if stale and not self._confirm( + "Samples at an Old Mass Tolerance", + f"{', '.join(stale)} still {'has' if len(stale) == 1 else 'have'} " + "EICs from an earlier mass tolerance. Export anyway?", + "Their CDF files could not be read when the tolerance changed, so " + "their values do not match the other samples. Put the files back " + "and change the tolerance again to rebuild them.", + "Export", + ): + return + file_path, _ = QFileDialog.getSaveFileName( self, "Export Data to Excel", diff --git a/src/manic/validation/peak_verdict.py b/src/manic/validation/peak_verdict.py index 90e3224c..25192d9e 100644 --- a/src/manic/validation/peak_verdict.py +++ b/src/manic/validation/peak_verdict.py @@ -14,6 +14,7 @@ class PeakVerdict(StrEnum): FAIL = "fail" ACCEPTED = "accepted" REJECTED = "rejected" + STALE = "stale" def resolve_verdict(meets_threshold: bool, review: PeakReview | None) -> PeakVerdict: @@ -29,4 +30,5 @@ def resolve_verdict(meets_threshold: bool, review: PeakReview | None) -> PeakVer PeakVerdict.FAIL: "#FFCCCC", PeakVerdict.ACCEPTED: "#E5D4F1", PeakVerdict.REJECTED: "#E0C9A6", + PeakVerdict.STALE: "#FFE699", } diff --git a/tests/test_export_warns_about_stale_samples.py b/tests/test_export_warns_about_stale_samples.py new file mode 100644 index 00000000..50f8db48 --- /dev/null +++ b/tests/test_export_warns_about_stale_samples.py @@ -0,0 +1,56 @@ +import openpyxl +import pytest +from PySide6.QtWidgets import QDialog + +import manic.ui.main_window as main_window_module +from manic.io.eic_importer import regenerate_all_eics_with_mass_tolerance +from manic.models.database import get_connection +from manic.ui.export_options_dialog import ExportOptionsDialog + +from conftest import _make_window +from test_restore_after_mass_tolerance_change import _write_cdf + + +@pytest.mark.parametrize("accept", [False, True], ids=["declined", "accepted"]) +def test_export_with_a_stale_sample_asks_before_writing( + qapp, tmp_path, monkeypatch, accept +): + with get_connection() as conn: + conn.execute( + "INSERT INTO compounds (compound_name, retention_time, mass0, " + "label_atoms, formula) VALUES ('Pyruvate', 5.0, 100.0, 1, 'C6H12O3NSi')" + ) + for sample in ("s1", "s2"): + _write_cdf(tmp_path / f"{sample}.cdf") + conn.execute( + "INSERT INTO samples (sample_name, file_name) VALUES (?, ?)", + (sample, str(tmp_path / f"{sample}.cdf")), + ) + regenerate_all_eics_with_mass_tolerance(0.2) + (tmp_path / "s2.cdf").rename(tmp_path / "s2.moved") + regenerate_all_eics_with_mass_tolerance(0.3) + + out = tmp_path / "out.xlsx" + window = _make_window(monkeypatch) + locations = [] + monkeypatch.setattr(ExportOptionsDialog, "exec", lambda d: QDialog.Accepted) + monkeypatch.setattr( + main_window_module.QFileDialog, + "getSaveFileName", + lambda *a, **k: locations.append(out) or (str(out), ""), + ) + monkeypatch.setattr(window, "_confirm", lambda *a, **k: accept) + try: + window.export_data() + if window._export_thread is not None: + assert window._export_thread.wait(30000) + qapp.processEvents() + + if accept: + raw = openpyxl.load_workbook(out)["Raw Values"] + assert [raw["B5"].value, raw["B6"].value] == ["s1", "s2"] + else: + assert locations == [] + assert not out.exists() + finally: + window.close() diff --git a/tests/test_stale_sample_marking.py b/tests/test_stale_sample_marking.py new file mode 100644 index 00000000..495ba0a7 --- /dev/null +++ b/tests/test_stale_sample_marking.py @@ -0,0 +1,102 @@ +import sqlite3 +from pathlib import Path + +import openpyxl +import pytest + +from manic.io.data_exporter import DataExporter +from manic.io.eic_importer import regenerate_all_eics_with_mass_tolerance +from manic.io.sample_reader import list_stale_samples +from manic.models import database +from manic.models.analysis import AnalysisMode +from manic.models.database import get_connection +from conftest import SCHEMA +from test_restore_after_mass_tolerance_change import _write_cdf + + +def _raw_value_fills(path): + raw = openpyxl.load_workbook(path)["Raw Values"] + return { + row[1].value: {cell.fill.fgColor.rgb for cell in row[2:]} + for row in raw.iter_rows(min_row=5) + } + + +def test_export_marks_a_sample_whose_cdf_failed_the_tolerance_reload(tmp_path): + with get_connection() as conn: + conn.execute( + "INSERT INTO compounds (compound_name, retention_time, mass0, " + "label_atoms, formula) VALUES ('Pyruvate', 5.0, 100.0, 1, 'C6H12O3NSi')" + ) + for sample in ("s1", "s2"): + _write_cdf(tmp_path / f"{sample}.cdf") + conn.execute( + "INSERT INTO samples (sample_name, file_name) VALUES (?, ?)", + (sample, str(tmp_path / f"{sample}.cdf")), + ) + regenerate_all_eics_with_mass_tolerance(0.2) + + (tmp_path / "s2.cdf").rename(tmp_path / "s2.moved") + regenerate_all_eics_with_mass_tolerance(0.3) + assert DataExporter(AnalysisMode.LABELLED).export_to_excel(str(tmp_path / "a.xlsx")) + assert _raw_value_fills(tmp_path / "a.xlsx") == { + "s1": {"00000000"}, + "s2": {"FFFFE699"}, + } + + (tmp_path / "s2.moved").rename(tmp_path / "s2.cdf") + regenerate_all_eics_with_mass_tolerance(0.3) + assert DataExporter(AnalysisMode.LABELLED).export_to_excel(str(tmp_path / "b.xlsx")) + assert _raw_value_fills(tmp_path / "b.xlsx") == { + "s1": {"00000000"}, + "s2": {"00000000"}, + } + + +def test_database_from_before_stale_tracking_loads_with_no_stale_samples( + tmp_path, monkeypatch +): + db_path = tmp_path / "old.db" + monkeypatch.setattr(database, "DB_FILE", db_path) + old_schema = SCHEMA.read_text(encoding="utf-8").replace( + " eics_stale INTEGER DEFAULT 0,\n", "" + ) + with sqlite3.connect(db_path) as conn: + conn.executescript(old_schema) + conn.execute("INSERT INTO samples (sample_name, file_name) VALUES ('s1', 'x')") + + database.init_db() + + assert list_stale_samples() == [] + + +def test_a_reload_that_stops_partway_leaves_unreached_samples_marked( + tmp_path, monkeypatch +): + with get_connection() as conn: + conn.execute( + "INSERT INTO compounds (compound_name, retention_time, mass0, " + "label_atoms, formula) VALUES ('Pyruvate', 5.0, 100.0, 1, 'C6H12O3NSi')" + ) + for sample in ("s1", "s2", "s3", "s4"): + _write_cdf(tmp_path / f"{sample}.cdf") + conn.execute( + "INSERT INTO samples (sample_name, file_name) VALUES (?, ?)", + (sample, str(tmp_path / f"{sample}.cdf")), + ) + regenerate_all_eics_with_mass_tolerance(0.2) + + (tmp_path / "s2.cdf").rename(tmp_path / "s2.moved") + # chmod cannot make a directory unreadable on Windows CI, so patch the check. + real_exists = Path.exists + + def exists(path, *args, **kwargs): + if path.name == "s3.cdf": + raise PermissionError(13, "Permission denied", str(path)) + return real_exists(path, *args, **kwargs) + + monkeypatch.setattr(Path, "exists", exists) + with pytest.raises(PermissionError): + regenerate_all_eics_with_mass_tolerance(0.3) + + assert list_stale_samples() == ["s2", "s3", "s4"]