From cdfb3aa2d30c080008c9e30cf6892a4d3884edd3 Mon Sep 17 00:00:00 2001 From: gitgat <1185276+gitgat@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:28:28 +0000 Subject: [PATCH 1/2] tools/export.py: copy its own files verbatim; the export changes Markdown prose only The first CI run here failed: the "blank lines around a dropped region doubled" mutant survived. The export had applied its drop-region transform to tests/test_export.py itself, so this copy of the test lost the fixture region that kills that mutant. - The tool, its lint and their tests are copied verbatim; the dev-link check is one helper. - A new test checks that an export changes Markdown prose only, so the published tests are the tests the tool was developed against. This restores the dropped fixture. Mutants: export 13/13. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++++ docs/tdd/export-red.txt | 6 ++++++ tests/mutants/export.json | 16 ++++++++-------- tests/test_export.py | 32 ++++++++++++++++++++++++++++++++ tools/export.py | 22 ++++++++++++++++------ 5 files changed, 66 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 192c41c..eb97349 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes are listed here. Versions follow [Semantic Versioning](https ### Added +- `tools/export.py` copies its own files (the tool, its lint and their tests) verbatim, and a + test checks that exporting the repo changes Markdown prose only. The export had dropped a + fixture from the public copy of its own tests, which let a mutant live there. + - CI runs install.sh's dry runs under bash 3.2 on Linux (`bash32`, the `bash:3.2` image pinned by digest) on every PR. The `macos` job and the `mutants` matrix are off while the repo is private, and `ci-ok` accepts their skip only then. `tools/check-workflows.py` fails a macOS diff --git a/docs/tdd/export-red.txt b/docs/tdd/export-red.txt index 1f7ac71..11c0e94 100644 --- a/docs/tdd/export-red.txt +++ b/docs/tdd/export-red.txt @@ -21,3 +21,9 @@ FAILED (failures=1) The tool describing the clone-path form in its own docs (before the MAY_NAME exemption): FAIL: test_the_tool_itself_may_describe_the_clone_path_form + +The tool's own files transformed (found by the public repo's first CI run: the export dropped a +drop-region fixture from tests/test_export.py, and the "blank lines around a dropped region +doubled" mutant survived there). The new tests on the old export.py: +FAIL: test_exporting_this_repo_leaves_the_tools_files_unchanged +FAIL: test_the_tools_own_files_are_copied_verbatim diff --git a/tests/mutants/export.json b/tests/mutants/export.json index ddfcbd4..5212031 100644 --- a/tests/mutants/export.json +++ b/tests/mutants/export.json @@ -29,7 +29,7 @@ }, { "name": "an unknown citation form passes", - "old": " if PLACEHOLDER in text and rel not in MAY_NAME:", + "old": " if PLACEHOLDER in text:", "new": " if False:" }, { @@ -49,8 +49,8 @@ }, { "name": "a dev-repo PR or issue link is let through", - "old": " if DEV_LINK.search(text):", - "new": " if False:" + "old": " if DEV_LINK.search(text):\n raise ValueError(", + "new": " if False:\n raise ValueError(" }, { "name": "only PR links are caught, not issues", @@ -59,18 +59,18 @@ }, { "name": "a plain-text lesson citation is let through", - "old": " if rel not in MAY_NAME and PLAIN_CITATION.search(text):", + "old": " if PLAIN_CITATION.search(text):", "new": " if False:" }, { "name": "a local clone path is let through", - "old": " if rel not in MAY_NAME and CLONE_PATH.search(text):", + "old": " if CLONE_PATH.search(text):", "new": " if False:" }, { - "name": "the tool may not describe the clone path form", - "old": " if rel not in MAY_NAME and CLONE_PATH.search(text):", - "new": " if CLONE_PATH.search(text):" + "name": "the tool's own files are transformed", + "old": " refuse_dev_links(rel, text)\n return text\n", + "new": " refuse_dev_links(rel, text)\n" } ] } diff --git a/tests/test_export.py b/tests/test_export.py index 0c4bfae..f98dbac 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -58,6 +58,12 @@ def export(src, *args): | `page_command` | none | | `lessons_url` | `URL` | + + ## Lessons links + + Skills cite incidents as lessons by number. + + ## Next """ SETTINGS = """\ @@ -155,6 +161,32 @@ def test_the_tool_itself_may_describe_the_clone_path_form(self): r, _ = export(repo({"tools/export.py": "# fails on /" + "projects/\n"})) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + def test_the_tools_own_files_are_copied_verbatim(self): + # The export must not apply its transforms to itself: a drop region in a test fixture was + # once dropped from the public copy of this file, leaving a mutant alive there. + fixture = "x = \"\"\"\n\ngone?\n\n\"\"\"\n" + r, out = export(repo({"tests/test_export.py": fixture, "a.md": "kept\n"})) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual((out / "tests" / "test_export.py").read_text(), fixture) + + def test_exporting_this_repo_changes_markdown_only(self): + root = EXPORT.parent.parent + if subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], capture_output=True).returncode != 0: + self.skipTest("not a git checkout") + r, out = export(root) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + # The export rewrites Markdown prose only: code, tests, fixtures and data come through + # byte for byte, so the public tests are the dev repo's tests. + files = subprocess.run(["git", "-C", str(root), "ls-files", "-z"], capture_output=True, text=True).stdout.split("\0") + changed = [] + for rel in filter(None, files): + if rel.endswith(".md") or not (out / rel).is_file(): + continue + committed = subprocess.run(["git", "-C", str(root), "show", f"HEAD:{rel}"], capture_output=True).stdout + if (out / rel).read_bytes() != committed: + changed.append(rel) + self.assertEqual(changed, []) + def test_the_repo_and_home_placeholders_are_fine(self): r, _ = export(repo({"a.txt": "File \"/lib/checks.py\"; config in /.config/x\n"})) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) diff --git a/tools/export.py b/tools/export.py index 1f909d8..0bc65fe 100755 --- a/tools/export.py +++ b/tools/export.py @@ -33,6 +33,7 @@ PLACEHOLDER = "PLAYBOOK_" + "LESSONS_URL" LINK = r"\[(?:lessons?|see lessons?) [^\]]*\]\(" + PLACEHOLDER + r"#\d+\)" # Files that may name the placeholder: this tool, the lint that allows it, and their tests. +# They are copied verbatim (see transform). MAY_NAME = {"tools/export.py", "tools/lint-skills.py", "tests/test_export.py", "tests/test_lint_settings.py"} START, END = "", "" # A link to the dev repo's PRs or issues: those numbers don't exist in the public repo, and the @@ -45,8 +46,18 @@ DEV_LINK = re.compile(r"github\.com/[^/\s]+/crewkit(?:-dev)?/(?:pull|issues)/\d+") +def refuse_dev_links(rel, text): + if DEV_LINK.search(text): + raise ValueError(f"{rel}: links to a dev-repo PR or issue; move it into an export:drop region") + + def transform(rel, text): """The public text of one file, or raise ValueError naming what couldn't be handled.""" + if rel in MAY_NAME: + # The tool, its lint and their tests are copied verbatim: their fixtures hold the very + # forms the transforms remove, and transforming them would weaken the public tests. + refuse_dev_links(rel, text) + return text lines, out, dropping, squeeze = text.split("\n"), [], False, False for n, line in enumerate(lines, 1): if line.strip() == START: @@ -77,15 +88,14 @@ def transform(rel, text): text = re.sub(r"\s*" + LINK, "", text) # A citation that wrapped onto its own line leaves only its punctuation there: fold it back. text = re.sub(r"\n[ \t]*([.,;:])[ \t]*(?=\n)", r"\1", text) - if rel not in MAY_NAME and (START in text or END in text): + if START in text or END in text: raise ValueError(f"{rel}: an export marker is left in the text (it must be alone on its line)") - if rel not in MAY_NAME and PLAIN_CITATION.search(text): + if PLAIN_CITATION.search(text): raise ValueError(f"{rel}: cites a lesson by number in plain text; reword it or move it into an export:drop region") - if rel not in MAY_NAME and CLONE_PATH.search(text): + if CLONE_PATH.search(text): raise ValueError(f"{rel}: names a local clone path (/projects/...); write /... instead") - if DEV_LINK.search(text): - raise ValueError(f"{rel}: links to a dev-repo PR or issue; move it into an export:drop region") - if PLACEHOLDER in text and rel not in MAY_NAME: + refuse_dev_links(rel, text) # after the drop regions are gone + if PLACEHOLDER in text: raise ValueError(f"{rel}: names {PLACEHOLDER} in a form the export can't remove") return text From ae7fb10c07d55ab412ffd55a035e8ace0f00e59f Mon Sep 17 00:00:00 2001 From: gitgat <1185276+gitgat@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:37:11 +0000 Subject: [PATCH 2/2] CI: split the slow land-batch and landing-lib mutation specs so each fits the 20-minute job This PR's first run cancelled "mutation check landing-lib-ci.json" at 20m18s (no mutant survived). Those specs run tests/landing-lib.bats and tests/land-batch.bats per mutant, about a minute each on the runners, so: - landing-lib-ci (20) -> landing-lib-gates (6), -workflow (5), -ci (5), -inputs (4); - land-batch (20, 16 min) -> land-batch (10) and land-batch-merge (10); - landing-lib-verdict (12, 12 min) -> landing-lib-verdict (6) and landing-lib-model (6). Same mutants, target and suite; every spec is in the CI matrix (the lint checks it). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 5 ++ docs/testing.md | 3 +- tests/mutants/land-batch-merge.json | 61 ++++++++++++++++++++ tests/mutants/land-batch.json | 50 ----------------- tests/mutants/landing-lib-ci.json | 75 ------------------------- tests/mutants/landing-lib-gates.json | 41 ++++++++++++++ tests/mutants/landing-lib-inputs.json | 31 ++++++++++ tests/mutants/landing-lib-model.json | 41 ++++++++++++++ tests/mutants/landing-lib-verdict.json | 30 ---------- tests/mutants/landing-lib-workflow.json | 36 ++++++++++++ 10 files changed, 217 insertions(+), 156 deletions(-) create mode 100644 tests/mutants/land-batch-merge.json create mode 100644 tests/mutants/landing-lib-gates.json create mode 100644 tests/mutants/landing-lib-inputs.json create mode 100644 tests/mutants/landing-lib-model.json create mode 100644 tests/mutants/landing-lib-workflow.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b1816e..587afb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,9 +69,14 @@ jobs: - export.json - gh-write.json - land-batch.json + - land-batch-merge.json - landing-lib-ci.json + - landing-lib-gates.json + - landing-lib-inputs.json + - landing-lib-model.json - landing-lib-risk.json - landing-lib-verdict.json + - landing-lib-workflow.json - lane-hygiene.json - lane-land-checks.json - lane-land-config.json diff --git a/docs/testing.md b/docs/testing.md index 13b31bf..3108c89 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -109,7 +109,8 @@ doesn't occur exactly once in its target, so editing a guarded line shows the mu at lint time, not after a mutation run. Keep each spec well inside the job's `timeout-minutes`: when one grows past about ten minutes, split its mutants into specs by area, with the same target and suite, as lane-land's `-checks`, `-config`, `-gates`, `-verdict` and -`-merge` specs do. `tools/mutate.sh` (install.sh's mutants) runs in a job +`-merge` specs and the landing-lib and land-batch specs do. Those run two bats suites per mutant, +about a minute each on CI's runners, so keep them to about six mutants. `tools/mutate.sh` (install.sh's mutants) runs in a job of its own too, against `tests/install.bats` only, the suite that runs `install.sh`. ## The checks invariant diff --git a/tests/mutants/land-batch-merge.json b/tests/mutants/land-batch-merge.json new file mode 100644 index 0000000..2963c77 --- /dev/null +++ b/tests/mutants/land-batch-merge.json @@ -0,0 +1,61 @@ +{ + "target": "plugins/crew-orchestrator/bin/land-batch", + "suite": [ + "bats", + "--abort", + "tests/landing-lib.bats", + "tests/land-batch.bats" + ], + "mutants": [ + { + "name": "squash not pinned to the verified head", + "old": "--squash --match-head-commit \"${HEAD[$pr]}\"", + "new": "--squash" + }, + { + "name": "parent check skipped", + "old": " [[ $(git -C \"$WT\" rev-parse \"$mc^\" 2>/dev/null) == \"$prev\" ]] ||", + "new": " true ||" + }, + { + "name": "tree check skipped", + "old": " [[ $(git -C \"$WT\" rev-parse \"$mc^{tree}\") == \"${TREE[$pr]}\" ]] ||", + "new": " true ||" + }, + { + "name": "main not re-checked before each squash", + "old": " main_is \"$prev\" || stop_merge 7", + "new": " true || stop_merge 7" + }, + { + "name": "no recheck before merging", + "old": " if [[ -n $PROB || $PR_HEAD != \"${HEAD[$pr]}\" ]]; then", + "new": " if false; then" + }, + { + "name": "up-to-date refusal retried as a generic failure", + "old": " stop_merge 10 \"#$pr REFUSED", + "new": " stop_merge 8 \"#$pr REFUSED" + }, + { + "name": "partial landing not reported", + "old": " if ((${#LANDED[@]})); then loud_partial \"$why\" \"$*\"; else", + "new": " if false; then loud_partial \"$why\" \"$*\"; else" + }, + { + "name": "a normal exit 20 reported as a partial landing", + "old": "FINISHED=1\n", + "new": "true\n" + }, + { + "name": "an unchecked fetch after a squash", + "old": " git -C \"$WT\" fetch -q origin \"$MAIN\" ||\n stop_merge 12 \"cannot fetch", + "new": " git -C \"$WT\" fetch -q origin \"$MAIN\" ||\n true || stop_merge 12 \"cannot fetch" + }, + { + "name": "PR heads aren't re-checked before the squash", + "old": " [[ $c == success ]] || stop_merge 9 \"#$pr: CI on its head is no longer green", + "new": " true || stop_merge 9 \"#$pr: CI on its head is no longer green" + } + ] +} diff --git a/tests/mutants/land-batch.json b/tests/mutants/land-batch.json index ad5420e..a62cb1a 100644 --- a/tests/mutants/land-batch.json +++ b/tests/mutants/land-batch.json @@ -7,31 +7,6 @@ "tests/land-batch.bats" ], "mutants": [ - { - "name": "squash not pinned to the verified head", - "old": "--squash --match-head-commit \"${HEAD[$pr]}\"", - "new": "--squash" - }, - { - "name": "parent check skipped", - "old": " [[ $(git -C \"$WT\" rev-parse \"$mc^\" 2>/dev/null) == \"$prev\" ]] ||", - "new": " true ||" - }, - { - "name": "tree check skipped", - "old": " [[ $(git -C \"$WT\" rev-parse \"$mc^{tree}\") == \"${TREE[$pr]}\" ]] ||", - "new": " true ||" - }, - { - "name": "main not re-checked before each squash", - "old": " main_is \"$prev\" || stop_merge 7", - "new": " true || stop_merge 7" - }, - { - "name": "no recheck before merging", - "old": " if [[ -n $PROB || $PR_HEAD != \"${HEAD[$pr]}\" ]]; then", - "new": " if false; then" - }, { "name": "high-risk PRs batched with the rest", "old": " if [[ -n $RISK ]]; then\n AREA[$pr]=$RISK; ALONE+=(\"$pr\")", @@ -62,26 +37,11 @@ "old": " if [[ -s $WT/.git/lb-seen && -n $(comm -12 \"$WT/.git/lb-seen\" \"$WT/.git/lb-cur\") ]]; then", "new": " if false; then" }, - { - "name": "up-to-date refusal retried as a generic failure", - "old": " stop_merge 10 \"#$pr REFUSED", - "new": " stop_merge 8 \"#$pr REFUSED" - }, - { - "name": "partial landing not reported", - "old": " if ((${#LANDED[@]})); then loud_partial \"$why\" \"$*\"; else", - "new": " if false; then loud_partial \"$why\" \"$*\"; else" - }, { "name": "unreadable changed files fail open", "old": " { PROB=\"changed-files-unreadable\"; return; }", "new": " true" }, - { - "name": "a normal exit 20 reported as a partial landing", - "old": "FINISHED=1\n", - "new": "true\n" - }, { "name": "renames show only the new path", "old": "diff --no-renames -z --name-only", @@ -92,20 +52,10 @@ "old": "diff --no-renames -z --name-only", "new": "diff --no-renames --name-only" }, - { - "name": "an unchecked fetch after a squash", - "old": " git -C \"$WT\" fetch -q origin \"$MAIN\" ||\n stop_merge 12 \"cannot fetch", - "new": " git -C \"$WT\" fetch -q origin \"$MAIN\" ||\n true || stop_merge 12 \"cannot fetch" - }, { "name": "the sweep closes any labelled batch-shaped PR", "old": "[[ -n $n && $title == \"batch:\"* ]] || continue", "new": "[[ -n $n ]] || continue" - }, - { - "name": "PR heads aren't re-checked before the squash", - "old": " [[ $c == success ]] || stop_merge 9 \"#$pr: CI on its head is no longer green", - "new": " true || stop_merge 9 \"#$pr: CI on its head is no longer green" } ] } diff --git a/tests/mutants/landing-lib-ci.json b/tests/mutants/landing-lib-ci.json index 3b7e9f9..7c0fad3 100644 --- a/tests/mutants/landing-lib-ci.json +++ b/tests/mutants/landing-lib-ci.json @@ -7,61 +7,11 @@ "tests/land-batch.bats" ], "mutants": [ - { - "name": "merge pause exemption ignored", - "old": "(.mergePauseExempt // []) | map(tostring) | index($pr) != null' \"$CAMPAIGN\") != true", - "new": "(.mergePauseExempt // []) | map(tostring) | index($pr) != null' \"$CAMPAIGN\") != false" - }, - { - "name": "stale lane head ignored", - "old": " if [[ -n $lane && ${lane#*$'\\t'} != \"\" && ${lane#*$'\\t'} != \"$PR_HEAD\" ]]; then", - "new": " if false; then" - }, - { - "name": "lane head compared by prefix", - "old": "${lane#*$'\\t'} != \"$PR_HEAD\" ]]; then", - "new": "$PR_HEAD != \"${lane#*$'\\t'}\"* ]]; then" - }, - { - "name": "corrupt campaign state opens the gates", - "old": "if [[ ! -r $CAMPAIGN ]] || ! jq -e 'type == \"object\"' \"$CAMPAIGN\" >/dev/null 2>&1; then", - "new": "if false; then" - }, - { - "name": "merge pause ignored by default", - "old": "HONOR_PAUSE=$(crew_setting land.honor_merge_pause true)", - "new": "HONOR_PAUSE=$(crew_setting land.honor_merge_pause false)" - }, - { - "name": "old bash accepted", - "old": "crew_land_bash_ok() {\n (($1 > 4 || ($1 == 4 && $2 >= 4))) ||", - "new": "crew_land_bash_ok() {\n true ||" - }, - { - "name": "ci.workflow optional", - "old": " CI_WORKFLOW=$(crew_require ci.workflow) || exit 78", - "new": " CI_WORKFLOW=$(crew_setting ci.workflow .github/workflows/ci.yml)" - }, - { - "name": "any event counts by default", - "old": " CI_EVENT=$(crew_setting ci.event pull_request)", - "new": " CI_EVENT=$(crew_setting ci.event \"\")" - }, - { - "name": "the workflow isn't passed", - "old": "--required-check \"$CHECK\" --workflow \"$CI_WORKFLOW\" \\", - "new": "--required-check \"$CHECK\" --workflow \".github/workflows/x.yml\" \\" - }, { "name": "statuses aren't read", "old": " ! g api \"repos/$R/commits/$1/status?per_page=100\" > \"$d/statuses.json\"; then", "new": " ! echo '{\"statuses\":[]}' > \"$d/statuses.json\"; then" }, - { - "name": "ignored checks aren't passed", - "old": " --ignored \"$IGNORED\" --min-green", - "new": " --ignored \"\" --min-green" - }, { "name": "a failed read is green", "old": "\"$d/statuses.json\"; then\n rm -rf \"$d\"; echo \"0 \"; return 0", @@ -72,40 +22,15 @@ "old": " *) echo \"${att:-0} \" ;;", "new": " *) echo \"${att:-1} success\" ;;" }, - { - "name": "only ci.workflow's jobs are read", - "old": " for id in $(jq -r '.[].id' \"$d/runs.json\"); do", - "new": " for id in $(jq -r --arg wf \"$CI_WORKFLOW\" '.[] | select(.path == $wf) | .id' \"$d/runs.json\"); do" - }, { "name": "statuses read 30 per page", "old": "commits/$1/status?per_page=100\"", "new": "commits/$1/status\"" }, - { - "name": "jobs listed without filter=latest", - "old": "actions/runs/$id/jobs?filter=latest&per_page=100\"", - "new": "actions/runs/$id/jobs?per_page=100\"" - }, - { - "name": "ignored_checks may name the required check", - "old": " [[ $v != \"$CHECK\" ]] || { echo \"$TOOL: ignored_checks names", - "new": " true || { echo \"$TOOL: ignored_checks names" - }, - { - "name": "ignored_checks names aren't trimmed", - "old": " v=${v#\"${v%%[![:space:]]*}\"}; v=${v%\"${v##*[![:space:]]}\"}\n", - "new": "" - }, { "name": "a failed status read is ignored", "old": " ! g api \"repos/$R/commits/$1/status?per_page=100\" > \"$d/statuses.json\"; then", "new": " ! { g api \"repos/$R/commits/$1/status?per_page=100\" > \"$d/statuses.json\"; true; }; then" - }, - { - "name": "an App author isn't mapped to its [bot] login", - "old": " [[ $author != app/* ]] || author=\"${author#app/}[bot]\"\n", - "new": "" } ] } diff --git a/tests/mutants/landing-lib-gates.json b/tests/mutants/landing-lib-gates.json new file mode 100644 index 0000000..6e74ce4 --- /dev/null +++ b/tests/mutants/landing-lib-gates.json @@ -0,0 +1,41 @@ +{ + "target": "plugins/crew-orchestrator/lib/landing.sh", + "suite": [ + "bats", + "--abort", + "tests/landing-lib.bats", + "tests/land-batch.bats" + ], + "mutants": [ + { + "name": "merge pause exemption ignored", + "old": "(.mergePauseExempt // []) | map(tostring) | index($pr) != null' \"$CAMPAIGN\") != true", + "new": "(.mergePauseExempt // []) | map(tostring) | index($pr) != null' \"$CAMPAIGN\") != false" + }, + { + "name": "merge pause ignored by default", + "old": "HONOR_PAUSE=$(crew_setting land.honor_merge_pause true)", + "new": "HONOR_PAUSE=$(crew_setting land.honor_merge_pause false)" + }, + { + "name": "corrupt campaign state opens the gates", + "old": "if [[ ! -r $CAMPAIGN ]] || ! jq -e 'type == \"object\"' \"$CAMPAIGN\" >/dev/null 2>&1; then", + "new": "if false; then" + }, + { + "name": "old bash accepted", + "old": "crew_land_bash_ok() {\n (($1 > 4 || ($1 == 4 && $2 >= 4))) ||", + "new": "crew_land_bash_ok() {\n true ||" + }, + { + "name": "stale lane head ignored", + "old": " if [[ -n $lane && ${lane#*$'\\t'} != \"\" && ${lane#*$'\\t'} != \"$PR_HEAD\" ]]; then", + "new": " if false; then" + }, + { + "name": "lane head compared by prefix", + "old": "${lane#*$'\\t'} != \"$PR_HEAD\" ]]; then", + "new": "$PR_HEAD != \"${lane#*$'\\t'}\"* ]]; then" + } + ] +} diff --git a/tests/mutants/landing-lib-inputs.json b/tests/mutants/landing-lib-inputs.json new file mode 100644 index 0000000..85f3408 --- /dev/null +++ b/tests/mutants/landing-lib-inputs.json @@ -0,0 +1,31 @@ +{ + "target": "plugins/crew-orchestrator/lib/landing.sh", + "suite": [ + "bats", + "--abort", + "tests/landing-lib.bats", + "tests/land-batch.bats" + ], + "mutants": [ + { + "name": "ignored checks aren't passed", + "old": " --ignored \"$IGNORED\" --min-green", + "new": " --ignored \"\" --min-green" + }, + { + "name": "ignored_checks may name the required check", + "old": " [[ $v != \"$CHECK\" ]] || { echo \"$TOOL: ignored_checks names", + "new": " true || { echo \"$TOOL: ignored_checks names" + }, + { + "name": "ignored_checks names aren't trimmed", + "old": " v=${v#\"${v%%[![:space:]]*}\"}; v=${v%\"${v##*[![:space:]]}\"}\n", + "new": "" + }, + { + "name": "an App author isn't mapped to its [bot] login", + "old": " [[ $author != app/* ]] || author=\"${author#app/}[bot]\"\n", + "new": "" + } + ] +} diff --git a/tests/mutants/landing-lib-model.json b/tests/mutants/landing-lib-model.json new file mode 100644 index 0000000..83963f2 --- /dev/null +++ b/tests/mutants/landing-lib-model.json @@ -0,0 +1,41 @@ +{ + "target": "plugins/crew-orchestrator/lib/landing.sh", + "suite": [ + "bats", + "--abort", + "tests/landing-lib.bats", + "tests/land-batch.bats" + ], + "mutants": [ + { + "name": "comment verdicts skip review_model", + "old": "--require-sha \"$REQUIRE_SHA\" --model \"$REVIEW_MODEL\" 2>&1)", + "new": "--require-sha \"$REQUIRE_SHA\" --model \"\" 2>&1)" + }, + { + "name": "record verdicts skip review_model", + "old": "--reviewers \"$REVIEWERS\" --model \"$REVIEW_MODEL\")", + "new": "--reviewers \"$REVIEWERS\" --model \"\")" + }, + { + "name": "review_model defaults to no model", + "old": " REVIEW_MODEL=$(crew_setting review_model opus)", + "new": " REVIEW_MODEL=$(crew_setting review_model \"\")" + }, + { + "name": "an empty review_model is ignored", + "old": " REVIEW_MODEL=$(crew_setting review_model opus)", + "new": " REVIEW_MODEL=opus" + }, + { + "name": "an unnamed author is accepted", + "old": " [[ -n $author ]] || { VWHY=\"the PR's author can't be named\"; return 1; }", + "new": " true" + }, + { + "name": "settings read from wherever the lander started", + "old": " crew_cd_main_checkout || crew_land_config_error \"$TOOL: can't find", + "new": " true || crew_land_config_error \"$TOOL: can't find" + } + ] +} diff --git a/tests/mutants/landing-lib-verdict.json b/tests/mutants/landing-lib-verdict.json index ad44d80..7aec4cb 100644 --- a/tests/mutants/landing-lib-verdict.json +++ b/tests/mutants/landing-lib-verdict.json @@ -36,36 +36,6 @@ "name": "verdict.py's refusal ignored", "old": " ((rc == 0)) && return 0\n VWHY=${out#not accepted: }", "new": " return 0\n VWHY=${out#not accepted: }" - }, - { - "name": "comment verdicts skip review_model", - "old": "--require-sha \"$REQUIRE_SHA\" --model \"$REVIEW_MODEL\" 2>&1)", - "new": "--require-sha \"$REQUIRE_SHA\" --model \"\" 2>&1)" - }, - { - "name": "record verdicts skip review_model", - "old": "--reviewers \"$REVIEWERS\" --model \"$REVIEW_MODEL\")", - "new": "--reviewers \"$REVIEWERS\" --model \"\")" - }, - { - "name": "review_model defaults to no model", - "old": " REVIEW_MODEL=$(crew_setting review_model opus)", - "new": " REVIEW_MODEL=$(crew_setting review_model \"\")" - }, - { - "name": "an empty review_model is ignored", - "old": " REVIEW_MODEL=$(crew_setting review_model opus)", - "new": " REVIEW_MODEL=opus" - }, - { - "name": "an unnamed author is accepted", - "old": " [[ -n $author ]] || { VWHY=\"the PR's author can't be named\"; return 1; }", - "new": " true" - }, - { - "name": "settings read from wherever the lander started", - "old": " crew_cd_main_checkout || crew_land_config_error \"$TOOL: can't find", - "new": " true || crew_land_config_error \"$TOOL: can't find" } ] } diff --git a/tests/mutants/landing-lib-workflow.json b/tests/mutants/landing-lib-workflow.json new file mode 100644 index 0000000..03030c3 --- /dev/null +++ b/tests/mutants/landing-lib-workflow.json @@ -0,0 +1,36 @@ +{ + "target": "plugins/crew-orchestrator/lib/landing.sh", + "suite": [ + "bats", + "--abort", + "tests/landing-lib.bats", + "tests/land-batch.bats" + ], + "mutants": [ + { + "name": "ci.workflow optional", + "old": " CI_WORKFLOW=$(crew_require ci.workflow) || exit 78", + "new": " CI_WORKFLOW=$(crew_setting ci.workflow .github/workflows/ci.yml)" + }, + { + "name": "any event counts by default", + "old": " CI_EVENT=$(crew_setting ci.event pull_request)", + "new": " CI_EVENT=$(crew_setting ci.event \"\")" + }, + { + "name": "the workflow isn't passed", + "old": "--required-check \"$CHECK\" --workflow \"$CI_WORKFLOW\" \\", + "new": "--required-check \"$CHECK\" --workflow \".github/workflows/x.yml\" \\" + }, + { + "name": "only ci.workflow's jobs are read", + "old": " for id in $(jq -r '.[].id' \"$d/runs.json\"); do", + "new": " for id in $(jq -r --arg wf \"$CI_WORKFLOW\" '.[] | select(.path == $wf) | .id' \"$d/runs.json\"); do" + }, + { + "name": "jobs listed without filter=latest", + "old": "actions/runs/$id/jobs?filter=latest&per_page=100\"", + "new": "actions/runs/$id/jobs?per_page=100\"" + } + ] +}