From b552915da0924dc49ea1a46f73bdd3bfcf48a872 Mon Sep 17 00:00:00 2001 From: fjh2333 <821097083@qq.com> Date: Thu, 29 Oct 2026 20:25:36 +0800 Subject: [PATCH] feat(qmd): tree-sitter AST break points for code files (P6.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the `ast` feature (default on) with `chunk::ast`: language detection for upstream's 11 extensions, LANGUAGE_QUERIES copied verbatim, upstream scores and strictly-highest dedup per position. Grammars are pinned to the exact versions upstream installs (typescript 0.23.2 — also used for JavaScript — python/go 0.25.0, rust 0.24.0) on the tree-sitter 0.26 runtime. Positions are converted from tree-sitter byte offsets to UTF-16 indices, which is what web-tree-sitter's startIndex reports, so non-ASCII files match too. A new upstream-recorded golden (18 files: every extension and capture, invalid syntax, empty, CJK, emoji, markdown, unknown extension) replays with zero mismatches. qmd-cli/qmd-mcp now depend on qmd with explicit features so `--no-default-features` really drops tree-sitter. Also folds in the remaining P5 test gaps: named-index daemon data, named + default daemons side by side, --daemon over stale and foreign pidfiles, single EOF breadcrumb, EPIPE on stderr, encodeQmdPath. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- CHANGELOG.md | 3 + Cargo.lock | 71 +++ crates/qmd-cli/Cargo.toml | 5 +- crates/qmd-cli/tests/mcp_daemon.rs | 271 ++++++++++ crates/qmd-cli/tests/mcp_stdio.rs | 139 +++++ crates/qmd-mcp/Cargo.toml | 2 +- crates/qmd-mcp/src/tools.rs | 20 + crates/qmd/Cargo.toml | 16 +- crates/qmd/src/chunk.rs | 3 + crates/qmd/src/chunk/ast.rs | 604 ++++++++++++++++++++++ crates/qmd/tests/fixtures/ast_golden.json | 490 ++++++++++++++++++ scripts/gen_ast_golden.ts | 231 +++++++++ 12 files changed, 1851 insertions(+), 4 deletions(-) create mode 100644 crates/qmd-cli/tests/mcp_stdio.rs create mode 100644 crates/qmd/src/chunk/ast.rs create mode 100644 crates/qmd/tests/fixtures/ast_golden.json create mode 100644 scripts/gen_ast_golden.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index c97588f..db2f863 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) ### Added +- `qmd` crate `ast` feature (default on, forwarded by `qmd-cli`) — tree-sitter AST break points for code files, the engine half of `--chunk-strategy auto` (P6.1). New `chunk::ast` module ports upstream `src/ast.ts`: `detect_language` (11 extensions → 6 languages, `.jsx` shares the TSX grammar and `.js/.mjs/.cjs` the TypeScript grammar, `qmd://` virtual paths, case-insensitive), verbatim `LANGUAGE_QUERIES` compiled once per language (`OnceLock`), upstream `SCORE_MAP`/`ast:` kinds, strictly-highest-score dedup per position, never-panics empty-vec failure paths. `ast_break_points` returns **UTF-16** positions like web-tree-sitter's `startIndex` (Rust `start_byte` byte offsets are converted, so non-ASCII files match upstream verbatim). Grammar crates pinned to upstream's resolved npm versions (`tree-sitter-typescript` =0.23.2 — JavaScript intentionally absent, `tree-sitter-python`/`-go` =0.25.0, `tree-sitter-rust` =0.24.0, `tree-sitter` 0.26.13 on upstream's 0.26 runtime line); statically linked, so upstream's grammar-load-failure branch is unreachable. `extractSymbols` not ported (upstream Phase-2 stub, D3). `scripts/gen_ast_golden.ts` records upstream `getASTBreakPoints` over an 18-file corpus (all 11 extensions covering every capture name, invalid syntax, empty, CJK/emoji, non-code, `qmd://` path) → `ast_golden.json` replayed verbatim. Chunk-merge integration (`Auto` still degrades to `Regex`) and the `qmd status` AST block land in P6.2. +- P5 leftover test coverage: `mcp_daemon.rs` gains `--index` named-index data-serving, named+default daemons coexisting (separate pidfiles/ports, independent `mcp stop`), `--daemon` over a stale pidfile (rewritten) and over a live-foreign pidfile (foreign process unsignalled); new `mcp_stdio.rs` subprocess suite — EOF breadcrumb prints exactly once, stderr EPIPE still answers + exits 0; `tools.rs` `encode_qmd_path` unit test. + - P5 acceptance review: `qmd-mcp` gains an embeddable `HttpServer` handle mirroring upstream's `HttpServerHandle` (`server.ts:889-1183`) — `HttpServer::bind`/`port()`/`serve(shutdown_future)` let embedders learn the bound port (port 0) and drive shutdown, while `serve_http` remains the CLI convenience (`bind` + SIGINT/SIGTERM future); `HttpOptions` gains `allowed_origins`/`allowed_hosts` (explicit values win over `QMD_ALLOWED_*` env like upstream's `??`, `origin-guard.ts:98-99`). The `qmd://{+path}` decoder now follows JS `decodeURIComponent` semantics exactly — malformed escapes (`%ZZ`, lone `%`, `%FF`, truncated `%E2%82`) and invalid UTF-8 reject with upstream's recorded `-32603 "URI error"` JSON-RPC error instead of passing through literally. The golden fixture grows to 61 stdio responses + 56 HTTP cases across a 4-document corpus (new spaced-path file `notes/External Podcast/2023 April - Interview.md`): percent-encoded (`%2F`/`%20`), double-encoded (`%252F`, single-pass), bare-filename suffix matches for `resources/read` and `get`, the `Did you mean one of these?` suggestion branch, `multi_get` `Errors:` block and `[SKIPPED: …]` marker, edge queries (empty/special/unicode/long/stopwords), and HTTP `tools/call get`. New `crates/qmd-mcp/tests/http.rs` adds the parked-reranker concurrency test (other requests answer while a call blocks in inference), allowlist-override and port-zero tests; `server.rs` gains inflight-gate release tests (idle fast path, panic/error closure releases). - `qmd mcp --http --daemon` and `qmd mcp stop` (P5.4): the launcher respawns `current_exe` with stdin nulled and stdout+stderr truncated into `$XDG_CACHE_HOME/qmd/mcp[-].log` (`~/.cache/qmd` without XDG), writes `mcp[-].pid`, prints upstream's `Started on http://:/mcp (PID )` + `Logs: ` lines and exits — `INDEX_PATH` pins the resolved db so the child never re-parses `--index`. The child detaches its own session (`rustix setsid` behind an env marker, P5-D8 — `CommandExt::pre_exec` is `unsafe` and stays out of scope). `stop` walks upstream's three outcomes — `Not running (no PID file).`, `Cleaned up stale PID file (server was not running).`, `Stopped QMD MCP server (PID ).` — all exit 0; a recorded PID is only signalled after the cmdline check (`mcp-pid.ts` ported 1:1: `/proc//cmdline`, `ps -o args=` then `-o command=` fallback, the `qmd[.ts|.js]` regex). `--daemon` already-running refuses on stderr with exit 1; `qmd status` gains the green `MCP: running (PID )` line and silently cleans a stale pidfile; a foreground `--http` run unlinks the pidfile when it happens to name itself; `--index` scopes pid/log names (`#772`). New `qmd_mcp::daemon` module (state files, paths, lenient pidfile parse, cmdline identity, `sigterm`, `detach_daemon_child`) with 10 unit tests porting `mcp-pid.test.ts` — including a live symlinked `qmd`-named process upstream couldn't fake; new `mcp_daemon.rs` subprocess suite covers the full lifecycle, `--index` scoping, the `status` line and the log file. diff --git a/Cargo.lock b/Cargo.lock index 703337f..f6c4a2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1211,6 +1211,11 @@ dependencies = [ "thiserror", "time", "tracing", + "tree-sitter", + "tree-sitter-go", + "tree-sitter-python", + "tree-sitter-rust", + "tree-sitter-typescript", "ureq", "walkdir", ] @@ -1683,6 +1688,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "strsim" version = "0.11.1" @@ -1911,6 +1922,66 @@ dependencies = [ "valuable", ] +[[package]] +name = "tree-sitter" +version = "0.26.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ebdd3a5a7e28a1890b876fdbd0c3c0fe0a6336cffaa104f11b9f720c9daa29" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "serde_json", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-go" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0af592be68c579aa78a16846bd19422978c3c52e438523d45ff5d1bff1f9d4a" + +[[package]] +name = "tree-sitter-python" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-rust" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9b18034c684a2420722be8b2a91c9c44f2546b631c039edf575ccba8c61be1" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "typenum" version = "1.20.1" diff --git a/crates/qmd-cli/Cargo.toml b/crates/qmd-cli/Cargo.toml index db34cb0..5523c0e 100644 --- a/crates/qmd-cli/Cargo.toml +++ b/crates/qmd-cli/Cargo.toml @@ -22,12 +22,13 @@ doc = false # bin `qmd` and lib `qmd` would collide under target/doc/ workspace = true [features] -default = ["mcp"] +default = ["mcp", "ast"] mcp = ["dep:qmd-mcp", "dep:tokio"] +ast = ["qmd/ast"] [dependencies] clap = { version = "4.6.7", features = ["derive"] } -qmd = { version = "0.6.0", path = "../qmd" } +qmd = { version = "0.6.0", path = "../qmd", default-features = false, features = ["llama"] } qmd-mcp = { version = "0.6.0", path = "../qmd-mcp", optional = true } serde_json = { version = "1.0.154", features = ["preserve_order"] } signal-hook = "0.4.4" diff --git a/crates/qmd-cli/tests/mcp_daemon.rs b/crates/qmd-cli/tests/mcp_daemon.rs index eb528d2..147effb 100644 --- a/crates/qmd-cli/tests/mcp_daemon.rs +++ b/crates/qmd-cli/tests/mcp_daemon.rs @@ -100,6 +100,43 @@ fn alive(pid: i64) -> bool { .is_ok_and(|o| o.status.success() && !String::from_utf8_lossy(&o.stdout).trim().is_empty()) } +/// `ps -p -o args=` — the process's full command line. +fn args_of(pid: i64) -> String { + Command::new("ps") + .args(["-p", &pid.to_string(), "-o", "args="]) + .output() + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_owned()) + .unwrap_or_default() +} + +/// Best-effort SIGTERM for daemons a test launched — a failed assert +/// must not leak a setsid'd server. Drop only signals when the +/// recorded pid still looks like our `qmd` binary (a foreign process +/// that reused the pid is never touched); the guard is a no-op once +/// `mcp stop` removed the pidfile. +struct DaemonGuard { + pidfiles: Vec, +} + +impl DaemonGuard { + fn new(pidfiles: impl IntoIterator) -> Self { + Self { + pidfiles: pidfiles.into_iter().collect(), + } + } +} + +impl Drop for DaemonGuard { + fn drop(&mut self) { + for path in &self.pidfiles { + let pid = recorded_pid(path); + if pid > 0 && args_of(pid).contains("qmd") { + drop(Command::new("kill").arg(pid.to_string()).output()); + } + } + } +} + /// Full lifecycle (cli.test.ts "mcp http daemon" foreground+daemon /// flow): launch → Started/Logs lines + exit 0 → pidfile + truncated /// log → detached session (ppid 1) → serves traffic → `mcp stop` → @@ -116,6 +153,7 @@ fn daemon_lifecycle_start_serve_stop() { let out = env.run(&["mcp", "--http", "--daemon", "--port", &port]); assert_eq!(out.exit_code, 0, "stderr: {}", out.stderr); + let _guard = DaemonGuard::new([pid_path.clone()]); assert!( out.stdout .contains(&format!("Started on http://localhost:{port}/mcp (PID ")), @@ -168,6 +206,7 @@ fn daemon_already_running_refuses() { let out = env.run(&["mcp", "--http", "--daemon", "--port", &port]); assert_eq!(out.exit_code, 0, "stderr: {}", out.stderr); let pid = recorded_pid(&pid_path); + let _guard = DaemonGuard::new([pid_path]); assert!(wait_health(port.parse().unwrap(), Duration::from_secs(10))); let again = env.run(&["mcp", "--http", "--daemon", "--port", &port]); @@ -309,6 +348,7 @@ fn daemon_named_index_scopes_state_files() { ); assert!(named_pid.exists(), "named pidfile missing"); assert!(!default_pid.exists(), "default pidfile must not appear"); + let _guard = DaemonGuard::new([named_pid.clone()]); let pid = recorded_pid(&named_pid); assert!(wait_health(port.parse().unwrap(), Duration::from_secs(10))); @@ -361,6 +401,7 @@ fn status_reports_running_daemon() { let out = env.run(&["mcp", "--http", "--daemon", "--port", &port]); assert_eq!(out.exit_code, 0, "stderr: {}", out.stderr); let pid = recorded_pid(&pid_path); + let _guard = DaemonGuard::new([pid_path]); assert!(wait_health(port.parse().unwrap(), Duration::from_secs(10))); let status = env.run(&["status"]); @@ -381,3 +422,233 @@ fn status_reports_running_daemon() { assert_eq!(after.exit_code, 0, "stderr: {}", after.stderr); assert!(!after.stdout.contains("MCP:"), "stdout: {}", after.stdout); } + +/// `--index ` daemon serves that named index (cli.test.ts +/// "mcp http daemon" ~:2630): the named corpus carries a term absent +/// from the default corpus, and the daemon's REST /query finds it. +#[test] +fn daemon_serves_named_index_data() { + let env = CliEnv::new(); + // Default index — the shared corpus (no unique term). + fixtures_corpus(&env); + env.write_config("index", &collection_yaml("docs", &env.fixtures, "**/*.md")); + env.update(); + // Named index — a separate corpus with a unique searchable term. + let alt = env.root.join("alt-corpus"); + std::fs::create_dir_all(&alt).unwrap(); + std::fs::write(alt.join("u.md"), "# U\n\nqwixzilonic term\n").unwrap(); + env.write_config("alt", &collection_yaml("docs", &alt, "**/*.md")); + env.reindex("alt", "alt", false); + + let port = common::free_port().to_string(); + let out = env + .qmd_command( + &[ + "--index", "alt", "mcp", "--http", "--daemon", "--port", &port, + ], + false, + &[], + ) + .output() + .unwrap(); + assert!( + out.status.success(), + "stderr: {}", + String::from_utf8_lossy(&out.stderr) + ); + let _guard = DaemonGuard::new([daemon_paths(&env, "alt").0]); + assert!(wait_health(port.parse().unwrap(), Duration::from_secs(10))); + + // The named daemon answers the named index's term. + let reply = common::http_request( + port.parse().unwrap(), + "POST", + "/query", + &serde_json::Map::new(), + Some(r#"{"searches":[{"type":"lex","query":"qwixzilonic"}],"rerank":false}"#), + ); + assert_eq!(reply.status, 200, "{}", reply.body); + assert!( + reply.body.contains("qwixzilonic") || reply.body.contains("u.md"), + "{}", + reply.body + ); + + let stopped = env + .qmd_command(&["--index", "alt", "mcp", "stop"], false, &[]) + .output() + .unwrap(); + let stop_out = String::from_utf8_lossy(&stopped.stdout); + assert!( + stopped.status.success() && stop_out.contains("Stopped QMD MCP server (PID "), + "named daemon not stopped: {stop_out}" + ); +} + +/// Named and default daemons run side by side on different ports +/// (cli.test.ts ~:2785): both serve /health, each `mcp stop` only +/// kills its own pidfile and port. +#[test] +fn named_and_default_daemons_coexist() { + let env = CliEnv::new(); + fixtures_corpus(&env); + env.write_config("index", &collection_yaml("docs", &env.fixtures, "**/*.md")); + env.update(); + let alt = env.root.join("alt-corpus"); + std::fs::create_dir_all(&alt).unwrap(); + std::fs::write(alt.join("u.md"), "# U\n\nalt corpus doc\n").unwrap(); + env.write_config("alt", &collection_yaml("docs", &alt, "**/*.md")); + env.reindex("alt", "alt", false); + + let port_a = common::free_port().to_string(); + let port_b = common::free_port().to_string(); + let (pid_a, _) = daemon_paths(&env, "index"); + let (pid_b, _) = daemon_paths(&env, "alt"); + + let out_a = env + .qmd_command( + &["mcp", "--http", "--daemon", "--port", &port_a], + false, + &[], + ) + .output() + .unwrap(); + assert!(out_a.status.success()); + let out_b = env + .qmd_command( + &[ + "--index", "alt", "mcp", "--http", "--daemon", "--port", &port_b, + ], + false, + &[], + ) + .output() + .unwrap(); + assert!(out_b.status.success()); + let _guard = DaemonGuard::new([pid_a.clone(), pid_b.clone()]); + assert!(wait_health( + port_a.parse().unwrap(), + Duration::from_secs(10) + )); + assert!(wait_health( + port_b.parse().unwrap(), + Duration::from_secs(10) + )); + assert!(pid_a.exists() && pid_b.exists()); + assert_ne!(recorded_pid(&pid_a), recorded_pid(&pid_b)); + + // Stopping the named daemon leaves the default one serving. + let stopped_b = env + .qmd_command(&["--index", "alt", "mcp", "stop"], false, &[]) + .output() + .unwrap(); + assert!( + stopped_b.status.success() + && String::from_utf8_lossy(&stopped_b.stdout).contains("Stopped QMD MCP server (PID "), + "stopped_b: {}", + String::from_utf8_lossy(&stopped_b.stdout) + ); + assert!(!pid_b.exists()); + let health_a = common::http_request( + port_a.parse().unwrap(), + "GET", + "/health", + &serde_json::Map::new(), + None, + ); + assert_eq!(health_a.status, 200); + + let stopped_a = env + .qmd_command(&["mcp", "stop"], false, &[]) + .output() + .unwrap(); + assert!( + stopped_a.status.success() + && String::from_utf8_lossy(&stopped_a.stdout).contains("Stopped QMD MCP server (PID "), + "stopped_a: {}", + String::from_utf8_lossy(&stopped_a.stdout) + ); + assert!(!pid_a.exists()); +} + +/// `--daemon` with a stale pidfile (dead pid) starts and rewrites the +/// pidfile (cli.test.ts ~:2920). +#[test] +fn daemon_starts_over_stale_pidfile() { + let env = indexed_env(); + let (pid_path, _) = daemon_paths(&env, "index"); + std::fs::create_dir_all(pid_path.parent().unwrap()).unwrap(); + std::fs::write(&pid_path, "999999999\n").unwrap(); // dead-pid contents; writes the same text upstream records + + let port = common::free_port().to_string(); + let out = env + .qmd_command(&["mcp", "--http", "--daemon", "--port", &port], false, &[]) + .output() + .unwrap(); + assert!( + out.status.success(), + "stderr: {}", + String::from_utf8_lossy(&out.stderr) + ); + let _guard = DaemonGuard::new([pid_path.clone()]); + assert!(wait_health(port.parse().unwrap(), Duration::from_secs(10))); + let pid = recorded_pid(&pid_path); + assert!( + pid > 0 && pid != 999_999_999, + "pidfile not rewritten: {pid}" + ); + + let stopped = env + .qmd_command(&["mcp", "stop"], false, &[]) + .output() + .unwrap(); + assert!(stopped.status.success()); +} + +/// `--daemon` with a *live foreign* pid in the pidfile starts anyway +/// (the identity check refuses to signal a non-qmd process → treated +/// like a stale file, cli.test.ts ~:2964) — and the foreign process +/// is still alive afterwards. +#[test] +fn daemon_starts_over_foreign_pidfile() { + let env = indexed_env(); + let (pid_path, _) = daemon_paths(&env, "index"); + std::fs::create_dir_all(pid_path.parent().unwrap()).unwrap(); + let mut sleeper = Command::new("sleep").arg("300").spawn().unwrap(); + let sleeper_pid = i64::from(sleeper.id().cast_signed()); + std::fs::write(&pid_path, format!("{sleeper_pid}\n")).unwrap(); + + let port = common::free_port().to_string(); + let out = env + .qmd_command(&["mcp", "--http", "--daemon", "--port", &port], false, &[]) + .output() + .unwrap(); + assert!( + out.status.success(), + "stderr: {}", + String::from_utf8_lossy(&out.stderr) + ); + assert!(wait_health(port.parse().unwrap(), Duration::from_secs(10))); + let _guard = DaemonGuard::new([pid_path.clone()]); + let daemon_pid = recorded_pid(&pid_path); + assert!( + daemon_pid > 0 && daemon_pid != sleeper_pid, + "pidfile not rewritten: {daemon_pid}" + ); + + // The foreign process was never signalled. + assert!(alive(sleeper_pid), "foreign process was killed"); + drop(sleeper.kill()); + drop(sleeper.wait()); + + let stopped = env + .qmd_command(&["mcp", "stop"], false, &[]) + .output() + .unwrap(); + assert!(stopped.status.success()); + let stopped_out = String::from_utf8_lossy(&stopped.stdout); + assert!( + stopped_out.contains(&format!("Stopped QMD MCP server (PID {daemon_pid}).")), + "stdout: {stopped_out}" + ); +} diff --git a/crates/qmd-cli/tests/mcp_stdio.rs b/crates/qmd-cli/tests/mcp_stdio.rs new file mode 100644 index 0000000..5ae4153 --- /dev/null +++ b/crates/qmd-cli/tests/mcp_stdio.rs @@ -0,0 +1,139 @@ +#![allow( + unused_crate_dependencies, + reason = "each test binary only uses a subset of the package deps" +)] +#![allow( + clippy::tests_outside_test_module, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + reason = "idiomatic test-code patterns" +)] + +//! Subprocess stdio lifecycle — upstream `mcp-stdio-lifecycle.test.ts` +//! cases that need a real `qmd mcp` binary: (:86) a second stdin EOF +//! must not re-trigger shutdown logging, and (:188) stderr writes that +//! fail (EPIPE — the parent dropped the read end) do not block the +//! JSON-RPC response or clean shutdown. + +mod common; + +use std::io::{Read as _, Write as _}; +use std::process::Stdio; +use std::time::Duration; + +use common::{CliEnv, collection_yaml, fixtures_corpus}; + +fn indexed_env() -> CliEnv { + let env = CliEnv::new(); + fixtures_corpus(&env); + env.write_config("index", &collection_yaml("docs", &env.fixtures, "**/*.md")); + env.update(); + env +} + +/// Read a pipe with a hard deadline so a hung child can't wedge the +/// suite (upstream relies on vitest's test timeout instead). +#[allow( + clippy::disallowed_methods, + reason = "wait polling needs real-time retries; no runtime in the test process" +)] +fn read_with_deadline( + mut child: std::process::Child, + timeout: Duration, +) -> (Option, String, String) { + let mut stdout_pipe = child.stdout.take().unwrap(); + let mut stderr_pipe = child.stderr.take().unwrap(); + let out_thread = std::thread::spawn(move || { + let mut s = String::new(); + drop(stdout_pipe.read_to_string(&mut s)); + s + }); + let err_thread = std::thread::spawn(move || { + let mut s = String::new(); + drop(stderr_pipe.read_to_string(&mut s)); + s + }); + let deadline = std::time::Instant::now() + timeout; + let status = loop { + match child.try_wait().unwrap() { + Some(s) => break Some(s), + None if std::time::Instant::now() > deadline => { + drop(child.kill()); + break None; + } + None => std::thread::sleep(Duration::from_millis(50)), + } + }; + ( + status.map(|s| s.code().unwrap_or(-1)), + out_thread.join().unwrap(), + err_thread.join().unwrap(), + ) +} + +/// EOF → exactly one `Shutting down (stdin closed)...` on stderr, +/// exit 0 (mcp-stdio-lifecycle.test.ts:86 — the EOF path is idempotent; +/// upstream asserts a double-EOF does not double-log). +#[test] +fn stdio_eof_breadcrumb_prints_once() { + let env = indexed_env(); + let mut child = env.qmd_command(&["mcp"], false, &[]); + let mut spawned = child.spawn().unwrap(); + // One answered call so the server is mid-session, then EOF. + spawned + .stdin + .as_mut() + .unwrap() + .write_all(b"{\"jsonrpc\":\"2.0\",\"id\":0,\"method\":\"initialize\",\"params\":{\"protocolVersion\":\"2025-06-18\",\"capabilities\":{},\"clientInfo\":{\"name\":\"t\",\"version\":\"1\"}}}\n{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}\n{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"status\",\"arguments\":{}}}\n") + .unwrap(); + drop(spawned.stdin.take()); + let (code, _stdout, stderr) = read_with_deadline(spawned, Duration::from_secs(15)); + assert_eq!(code, Some(0), "stderr: {stderr}"); + let count = stderr.matches("Shutting down (stdin closed)...").count(); + assert_eq!(count, 1, "stderr: {stderr}"); +} + +/// stderr EPIPE (parent dropped the read end) — the server still +/// answers on stdout and exits 0 on EOF +/// (mcp-stdio-lifecycle.test.ts:188 — a throwing stderr must not +/// block shutdown). +#[test] +#[allow( + clippy::disallowed_methods, + reason = "wait polling needs real-time retries; no runtime in the test process" +)] +fn stdio_stderr_epipe_still_answers_and_exits() { + let env = indexed_env(); + let mut child = env.qmd_command(&["mcp"], false, &[]); + child.stderr(Stdio::piped()); + let mut spawned = child.spawn().unwrap(); + // Dropping the parent's read end makes every stderr write hit EPIPE. + drop(spawned.stderr.take()); + spawned + .stdin + .as_mut() + .unwrap() + .write_all(b"{\"jsonrpc\":\"2.0\",\"id\":0,\"method\":\"initialize\",\"params\":{\"protocolVersion\":\"2025-06-18\",\"capabilities\":{},\"clientInfo\":{\"name\":\"t\",\"version\":\"1\"}}}\n{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}\n{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"status\",\"arguments\":{}}}\n") + .unwrap(); + drop(spawned.stdin.take()); + let mut stdout_pipe = spawned.stdout.take().unwrap(); + let mut stdout = String::new(); + let deadline = std::time::Instant::now() + Duration::from_secs(15); + let code = loop { + match spawned.try_wait().unwrap() { + Some(s) => break s.code().unwrap_or(-1), + None if std::time::Instant::now() > deadline => { + drop(spawned.kill()); + panic!("child hung after EPIPE+EOF"); + } + None => std::thread::sleep(Duration::from_millis(50)), + } + }; + drop(stdout_pipe.read_to_string(&mut stdout)); + assert_eq!(code, 0); + assert!( + stdout.contains("\"id\":1") && stdout.contains("QMD Index Status"), + "stdout: {stdout}" + ); +} diff --git a/crates/qmd-mcp/Cargo.toml b/crates/qmd-mcp/Cargo.toml index 51fbc58..16bc121 100644 --- a/crates/qmd-mcp/Cargo.toml +++ b/crates/qmd-mcp/Cargo.toml @@ -18,7 +18,7 @@ workspace = true [dependencies] axum = { version = "0.8.9" } -qmd = { version = "0.6.0", path = "../qmd" } +qmd = { version = "0.6.0", path = "../qmd", default-features = false, features = ["llama"] } regex = { version = "1.13.1" } rmcp = { version = "3.5.1", features = ["server", "transport-io"] } rustix = { version = "1.1.5", features = ["process"] } diff --git a/crates/qmd-mcp/src/tools.rs b/crates/qmd-mcp/src/tools.rs index e3c7db7..9ac3df4 100644 --- a/crates/qmd-mcp/src/tools.rs +++ b/crates/qmd-mcp/src/tools.rs @@ -867,3 +867,23 @@ mod validate { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// `encode_qmd_path` (`vpath.rs:128`, upstream `encodeQmdPath`) — + /// per-segment `encodeURIComponent`: slashes preserved, spaces and + /// specials escaped (mcp.test.ts:832). + #[test] + fn encode_qmd_path_preserves_slashes_encodes_specials() { + assert_eq!( + encode_qmd_path("External Podcast/2023 April - Interview.md"), + "External%20Podcast/2023%20April%20-%20Interview.md" + ); + assert!(encode_qmd_path("a/b c.md").contains('/')); + assert!(encode_qmd_path("a/b c.md").contains("%20")); + // Hash fragments and `?` are escaped inside a segment too. + assert_eq!(encode_qmd_path("a?b#c.md"), "a%3Fb%23c.md"); + } +} diff --git a/crates/qmd/Cargo.toml b/crates/qmd/Cargo.toml index 7e157d5..5167d80 100644 --- a/crates/qmd/Cargo.toml +++ b/crates/qmd/Cargo.toml @@ -15,10 +15,16 @@ features = ["testing"] rustdoc-args = ["--cfg", "docsrs"] [features] -default = ["llama"] +default = ["llama", "ast"] # Deterministic in-memory inference backends (`qmd::testing`) for tests # and downstream embedders — no model files required. testing = [] +# Tree-sitter AST break points for `--chunk-strategy auto` +# (`chunk::ast`). Grammar crates are pinned to the exact upstream wasm +# versions; `tree-sitter` runtime on the 0.26 line matching upstream's +# `web-tree-sitter` 0.26. Enabled by default. +ast = ["dep:tree-sitter", "dep:tree-sitter-typescript", "dep:tree-sitter-python", + "dep:tree-sitter-go", "dep:tree-sitter-rust"] # On-device inference on llama.cpp (`llm::llamacpp`) plus HuggingFace model # download (`llm::download`). Enabled by default; builds C++ via cmake and # bindgen, so `cmake`, `clang`/`libclang` must be installed. @@ -52,6 +58,14 @@ time = { version = "0.3.36", features = ["formatting", "macros", "parsi llama-cpp-2 = { version = "=0.1.158", optional = true } ureq = { version = "3.4.2", optional = true } tracing = "0.1.44" +tree-sitter = { version = "0.26.13", optional = true } +# Grammar crates pinned `=` to the exact upstream versions +# (p6-ast.md §2.2). JavaScript intentionally absent — upstream reuses +# the TypeScript grammar for `.js` (ast.ts:69). +tree-sitter-typescript = { version = "=0.23.2", optional = true } +tree-sitter-python = { version = "=0.25.0", optional = true } +tree-sitter-go = { version = "=0.25.0", optional = true } +tree-sitter-rust = { version = "=0.24.0", optional = true } [dev-dependencies] tempfile = "3.23.0" diff --git a/crates/qmd/src/chunk.rs b/crates/qmd/src/chunk.rs index e43e09a..9d02ce8 100644 --- a/crates/qmd/src/chunk.rs +++ b/crates/qmd/src/chunk.rs @@ -25,6 +25,9 @@ use regex::Regex; use crate::llm::{CancelToken, Embedder, InferenceError}; +#[cfg(feature = "ast")] +pub mod ast; + /// Upstream `CHUNK_SIZE_TOKENS` (store.ts:114): 900 tokens per chunk. pub const CHUNK_SIZE_TOKENS: usize = 900; /// Upstream `CHUNK_OVERLAP_TOKENS` (store.ts:115): 15% of the chunk size. diff --git a/crates/qmd/src/chunk/ast.rs b/crates/qmd/src/chunk/ast.rs new file mode 100644 index 0000000..4b687af --- /dev/null +++ b/crates/qmd/src/chunk/ast.rs @@ -0,0 +1,604 @@ +//! Tree-sitter AST break points — upstream `src/ast.ts`. +//! +//! Position units: web-tree-sitter's `startIndex` (ast.ts:305) is a +//! **UTF-16 code-unit index** — the golden recording puts a CJK file's +//! `export` at 12 (UTF-16), not 28 (UTF-8 bytes). Rust +//! `Node::start_byte` gives byte offsets, so [`ast_break_points`] +//! converts each to a UTF-16 index; the result matches upstream +//! verbatim (no deviation). `break_points_utf8` keeps the raw byte +//! view internally. +//! +//! Grammars are statically linked (no wasm load), so upstream's +//! grammar-load-failure branch (`formatGrammarLoadError`, ast.ts:75) +//! is unreachable and not ported. `extractSymbols` is not ported — +//! upstream marks it a Phase-2 stub with no callers (D3). + +use std::collections::HashMap; +use std::sync::OnceLock; + +use tree_sitter::{Parser, Query, QueryCursor, StreamingIterator as _}; + +use super::BreakPoint; + +/// Supported code languages — upstream `SupportedLanguage` (ast.ts:34). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Language { + /// `.ts`, `.mts`, `.cts` (`tree-sitter-typescript.wasm`). + TypeScript, + /// `.tsx`, `.jsx` (`tree-sitter-tsx.wasm`). + Tsx, + /// `.js`, `.mjs`, `.cjs` — upstream reuses the TypeScript grammar + /// (ast.ts:69). + JavaScript, + /// `.py` (`tree-sitter-python.wasm`). + Python, + /// `.go` (`tree-sitter-go.wasm`). + Go, + /// `.rs` (`tree-sitter-rust.wasm`). + Rust, +} + +impl Language { + /// Upstream `SupportedLanguage` string. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::TypeScript => "typescript", + Self::Tsx => "tsx", + Self::JavaScript => "javascript", + Self::Python => "python", + Self::Go => "go", + Self::Rust => "rust", + } + } +} + +/// `detectLanguage` (ast.ts:54-58) — extension → language, mirroring +/// node `path.extname`: the last `.` after the last `/`, lowercase; +/// `.ts`-style dotfiles have no extension; `file.` yields `"."`. +/// +/// Works on `qmd://` virtual paths — only the basename matters. +#[must_use] +pub fn detect_language(path: &str) -> Option { + let name = path.rsplit('/').next().unwrap_or(path); + let dot = name.rfind('.')?; + if dot == 0 { + return None; + } + Some(match name[dot..].to_ascii_lowercase().as_str() { + ".ts" | ".mts" | ".cts" => Language::TypeScript, + ".tsx" | ".jsx" => Language::Tsx, + ".js" | ".mjs" | ".cjs" => Language::JavaScript, + ".py" => Language::Python, + ".go" => Language::Go, + ".rs" => Language::Rust, + _ => return None, + }) +} + +// LANGUAGE_QUERIES — verbatim copy of ast.ts:94-151. + +const Q_TYPESCRIPT: &str = r" + (export_statement) @export + (class_declaration) @class + (function_declaration) @func + (method_definition) @method + (interface_declaration) @iface + (type_alias_declaration) @type + (enum_declaration) @enum + (import_statement) @import + (lexical_declaration (variable_declarator value: (arrow_function))) @func + (lexical_declaration (variable_declarator value: (function_expression))) @func + "; + +const Q_TSX: &str = Q_TYPESCRIPT; + +const Q_JAVASCRIPT: &str = r" + (export_statement) @export + (class_declaration) @class + (function_declaration) @func + (method_definition) @method + (import_statement) @import + (lexical_declaration (variable_declarator value: (arrow_function))) @func + (lexical_declaration (variable_declarator value: (function_expression))) @func + "; + +const Q_PYTHON: &str = r" + (class_definition) @class + (function_definition) @func + (decorated_definition) @decorated + (import_statement) @import + (import_from_statement) @import + "; + +const Q_GO: &str = r" + (type_declaration) @type + (function_declaration) @func + (method_declaration) @method + (import_declaration) @import + "; + +const Q_RUST: &str = r" + (struct_item) @struct + (impl_item) @impl + (function_item) @func + (trait_item) @trait + (enum_item) @enum + (use_declaration) @import + (type_item) @type + (mod_item) @mod + "; + +/// `SCORE_MAP` (ast.ts:158-172); unlisted captures score 20. +fn capture_score(name: &str) -> u32 { + match name { + "class" | "iface" | "struct" | "trait" | "impl" | "mod" => 100, + "export" | "func" | "method" | "decorated" => 90, + "type" | "enum" => 80, + "import" => 60, + _ => 20, + } +} + +fn ts_language(lang: Language) -> tree_sitter::Language { + match lang { + Language::TypeScript | Language::JavaScript => { + tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into() + } + Language::Tsx => tree_sitter_typescript::LANGUAGE_TSX.into(), + Language::Python => tree_sitter_python::LANGUAGE.into(), + Language::Go => tree_sitter_go::LANGUAGE.into(), + Language::Rust => tree_sitter_rust::LANGUAGE.into(), + } +} + +const fn query_source(lang: Language) -> &'static str { + match lang { + Language::TypeScript => Q_TYPESCRIPT, + Language::Tsx => Q_TSX, + Language::JavaScript => Q_JAVASCRIPT, + Language::Python => Q_PYTHON, + Language::Go => Q_GO, + Language::Rust => Q_RUST, + } +} + +/// Compiled-query cache — upstream `queryCache` (ast.ts:193). +fn query_for(lang: Language) -> Option<&'static Query> { + static TYPESCRIPT: OnceLock> = OnceLock::new(); + static TSX: OnceLock> = OnceLock::new(); + static JAVASCRIPT: OnceLock> = OnceLock::new(); + static PYTHON: OnceLock> = OnceLock::new(); + static GO: OnceLock> = OnceLock::new(); + static RUST: OnceLock> = OnceLock::new(); + let slot = match lang { + Language::TypeScript => &TYPESCRIPT, + Language::Tsx => &TSX, + Language::JavaScript => &JAVASCRIPT, + Language::Python => &PYTHON, + Language::Go => &GO, + Language::Rust => &RUST, + }; + slot.get_or_init(|| Query::new(&ts_language(lang), query_source(lang)).ok()) + .as_ref() +} + +/// `getASTBreakPoints` (ast.ts:274-318) computed on `start_byte` +/// positions — UTF-8 byte offsets. Internal: the public surface +/// converts to upstream's UTF-16 indices; this view feeds the +/// byte-level parity check on ASCII corpus files. +pub(crate) fn break_points_utf8(content: &str, path: &str) -> Vec { + let Some(lang) = detect_language(path) else { + return Vec::new(); + }; + let Some(query) = query_for(lang) else { + return Vec::new(); + }; + let mut parser = Parser::new(); + if parser.set_language(&ts_language(lang)).is_err() { + return Vec::new(); + } + let Some(tree) = parser.parse(content, None) else { + return Vec::new(); + }; + // `captures()` yields captures in upstream `query.captures` order — + // the tie on equal score at a position keeps the first capture, + // so iteration order must match upstream's exactly (ast.ts:296-313). + let mut cursor = QueryCursor::new(); + let mut captures = cursor.captures(query, tree.root_node(), content.as_bytes()); + let mut seen: HashMap = HashMap::new(); + while let Some((m, cap_idx)) = captures.next() { + let Some(cap) = m.captures.get(*cap_idx) else { + continue; + }; + let pos = cap.node.start_byte(); + let name = query + .capture_names() + .get(cap.index as usize) + .copied() + .unwrap_or_default(); + let score = capture_score(name); + if seen.get(&pos).is_none_or(|x| score > x.score) { + seen.insert( + pos, + BreakPoint { + pos, + score, + kind: format!("ast:{name}").into(), + }, + ); + } + } + let mut points: Vec = seen.into_values().collect(); + points.sort_by_key(|p| p.pos); + points +} + +/// `getASTBreakPoints` verbatim — positions as UTF-16 code-unit +/// indices (upstream's `startIndex` semantics). Returns empty +/// for unsupported languages, parse failures, or query errors. +#[must_use] +pub fn ast_break_points(content: &str, path: &str) -> Vec { + let mut points = break_points_utf8(content, path); + // Byte positions land on char boundaries (tree-sitter node + // starts), so one char_indices pass converts them all. + let mut byte = 0usize; + let mut utf16 = 0usize; + let mut chars = content.char_indices(); + for bp in &mut points { + while byte < bp.pos { + match chars.next() { + Some((i, c)) => { + byte = i + c.len_utf8(); + utf16 += c.len_utf16(); + } + None => break, + } + } + bp.pos = utf16; + } + points +} + +#[cfg(test)] +mod tests { + //! Ports of upstream `test/ast.test.ts` (detectLanguage :16-70, + //! per-language break points :74-292, error handling :295-318) plus + //! the byte → UTF-16 conversion and the `ast_golden.json` replay. + #![allow( + clippy::missing_assert_message, + reason = "idiomatic test-code patterns" + )] + + use super::*; + + // Sample sources copied verbatim from ast.test.ts fixtures. + + const TS_SAMPLE: &str = r"import { Database } from './db'; +import type { User } from './types'; + +interface AuthConfig { + secret: string; + ttl: number; +} + +type UserId = string; + +export class AuthService { + constructor(private db: Database) {} + + async authenticate(user: User, token: string): Promise { + const session = await this.db.findSession(token); + return session?.userId === user.id; + } + + validateToken(token: string): boolean { + return token.length === 64; + } +} + +export function hashPassword(password: string): string { + return crypto.createHash('sha256').update(password).digest('hex'); +} +"; + + const PY_SAMPLE: &str = r"import os +from typing import Optional + +class AuthService: + def __init__(self, db): + self.db = db + + async def authenticate(self, user, token): + session = await self.db.find(token) + return session.user_id == user.id + + def validate_token(self, token): + return len(token) == 64 + +def hash_password(password: str) -> str: + return hashlib.sha256(password.encode()).hexdigest() + +@decorator +def decorated_func(): + pass +"; + + const GO_SAMPLE: &str = r#"package main + +import "fmt" + +type AuthService struct { + db *Database +} + +func (s *AuthService) Authenticate(user User) bool { + return true +} + +func HashPassword(password string) string { + return "hash" +} +"#; + + const RS_SAMPLE: &str = r"use std::collections::HashMap; + +struct AuthService { + db: Database, +} + +impl AuthService { + fn authenticate(&self, user: &User) -> bool { + true + } +} + +trait Authenticatable { + fn validate(&self) -> bool; +} + +enum Role { + Admin, + User, +} + +fn hash_password(password: &str) -> String { + String::new() +} +"; + + // ---- detectLanguage (ast.test.ts:16-70) ---- + + #[test] + fn detects_all_extensions() { + let cases: &[(&str, Language)] = &[ + ("src/auth.ts", Language::TypeScript), + ("src/auth.mts", Language::TypeScript), + ("src/auth.cts", Language::TypeScript), + ("src/App.tsx", Language::Tsx), + ("src/util.js", Language::JavaScript), + ("src/util.mjs", Language::JavaScript), + ("src/util.cjs", Language::JavaScript), + ("src/App.jsx", Language::Tsx), + ("src/auth.py", Language::Python), + ("src/auth.go", Language::Go), + ("src/auth.rs", Language::Rust), + ]; + for (path, want) in cases { + assert_eq!(detect_language(path), Some(*want), "{path}"); + } + } + + #[test] + fn rejects_non_code_and_extensionless() { + for path in ["docs/README.md", "data/file.csv", "config.yaml", "Makefile"] { + assert_eq!(detect_language(path), None, "{path}"); + } + } + + #[test] + fn detection_is_case_insensitive() { + assert_eq!(detect_language("src/Auth.TS"), Some(Language::TypeScript)); + assert_eq!(detect_language("src/Auth.PY"), Some(Language::Python)); + } + + #[test] + fn detection_works_on_virtual_paths() { + assert_eq!( + detect_language("qmd://myproject/src/auth.ts"), + Some(Language::TypeScript) + ); + assert_eq!(detect_language("qmd://docs/README.md"), None); + } + + // ---- TypeScript (ast.test.ts:74-140) ---- + + #[test] + fn ts_produces_break_points_at_boundaries() { + let points = ast_break_points(TS_SAMPLE, "src/auth.ts"); + assert!(!points.is_empty()); + let has = |frag: &str| points.iter().any(|p| p.kind.contains(frag)); + assert!(has("import") && has("iface") && has("type")); + assert!(has("export") || has("class")); + assert!(has("method")); + } + + #[test] + fn ts_break_points_sorted() { + let points = ast_break_points(TS_SAMPLE, "src/auth.ts"); + for w in points.windows(2) { + let first = w.first().expect("window"); + let last = w.last().expect("window"); + assert!(last.pos >= first.pos); + } + } + + #[test] + fn ts_scores_align_with_hierarchy() { + let points = ast_break_points(TS_SAMPLE, "src/auth.ts"); + let score = |kind: &str| points.iter().find(|p| p.kind == kind).map(|p| p.score); + assert_eq!(score("ast:iface"), Some(100)); + assert_eq!(score("ast:method"), Some(90)); + assert_eq!(score("ast:import"), Some(60)); + } + + #[test] + fn ts_break_point_positions_match_content() { + let points = ast_break_points(TS_SAMPLE, "src/auth.ts"); + let first_import = points.iter().find(|p| p.kind == "ast:import").unwrap(); + assert_eq!( + TS_SAMPLE + .get(first_import.pos..first_import.pos + 6) + .unwrap_or_default(), + "import" + ); + } + + // ---- Python (ast.test.ts:144-181) ---- + + #[test] + fn py_produces_class_func_import_decorated() { + let points = ast_break_points(PY_SAMPLE, "auth.py"); + let has = |frag: &str| points.iter().any(|p| p.kind.contains(frag)); + assert!(has("import") && has("class") && has("func") && has("decorated")); + } + + #[test] + fn py_captures_methods_inside_classes() { + let points = ast_break_points(PY_SAMPLE, "auth.py"); + let funcs = points.iter().filter(|p| p.kind == "ast:func").count(); + assert!(funcs >= 3, "{funcs} func points"); + } + + // ---- Go (ast.test.ts:185-223) ---- + + #[test] + fn go_produces_type_func_method_import() { + let points = ast_break_points(GO_SAMPLE, "auth.go"); + let has = |frag: &str| points.iter().any(|p| p.kind.contains(frag)); + assert!(has("import") && has("type") && has("method") && has("func")); + } + + #[test] + fn go_func_and_method_score_90() { + let points = ast_break_points(GO_SAMPLE, "auth.go"); + let score = |kind: &str| points.iter().find(|p| p.kind == kind).map(|p| p.score); + assert_eq!(score("ast:func"), Some(90)); + assert_eq!(score("ast:method"), Some(90)); + } + + // ---- Rust (ast.test.ts:227-292) ---- + + #[test] + fn rs_produces_all_node_kinds() { + let points = ast_break_points(RS_SAMPLE, "auth.rs"); + let has = |frag: &str| points.iter().any(|p| p.kind.contains(frag)); + assert!(has("import") && has("struct") && has("impl")); + assert!(has("trait") && has("enum") && has("func")); + } + + #[test] + fn rs_struct_impl_trait_score_100() { + let points = ast_break_points(RS_SAMPLE, "auth.rs"); + let score = |kind: &str| points.iter().find(|p| p.kind == kind).map(|p| p.score); + assert_eq!(score("ast:struct"), Some(100)); + assert_eq!(score("ast:impl"), Some(100)); + assert_eq!(score("ast:trait"), Some(100)); + } + + // ---- Error handling (ast.test.ts:295-318, first 4) ---- + + #[test] + fn empty_for_unsupported_types() { + assert!(ast_break_points("# Hello World", "readme.md").is_empty()); + assert!(ast_break_points("data,here", "file.csv").is_empty()); + } + + #[test] + fn empty_content_parses_to_nothing() { + assert!(ast_break_points("", "empty.ts").is_empty()); + } + + #[test] + fn invalid_syntax_does_not_panic() { + // Tree-sitter is error-tolerant — a broken file yields a tree + // with ERROR nodes; result may be empty or partial, never a panic. + let _points = ast_break_points("function { broken syntax %%%", "broken.ts"); + } + + // ---- byte → UTF-16 conversion (upstream `startIndex` units) ---- + + #[test] + fn ascii_byte_positions_equal_utf16() { + let content = "import x;\n\nclass Foo {\n m() {}\n}\n"; + let utf8 = break_points_utf8(content, "a.ts"); + let utf16 = ast_break_points(content, "a.ts"); + assert_eq!(utf8.len(), utf16.len()); + for (a, b) in utf8.iter().zip(&utf16) { + assert_eq!(a.pos, b.pos); + assert_eq!(a.score, b.score); + assert_eq!(a.kind, b.kind); + } + } + + #[test] + fn cjk_and_astral_convert_to_utf16_indices() { + // 4-byte emoji + 3-byte CJK shift byte offsets well past the + // UTF-16 indices; conversion must land on JS-string positions. + let content = "// 日本語\nexport function f() { return \"🎉\"; }\n"; + let utf8 = break_points_utf8(content, "a.ts"); + let utf16 = ast_break_points(content, "a.ts"); + assert_eq!(utf8.len(), utf16.len()); + for (a, b) in utf8.iter().zip(&utf16) { + let want = content + .get(..a.pos) + .map_or(0, |s| s.chars().map(char::len_utf16).sum()); + assert_eq!(b.pos, want, "kind {}", b.kind); + assert!(a.pos > b.pos, "byte pos should exceed utf16 pos here"); + } + } + + // ---- Golden replay: upstream recordings are byte-authoritative ---- + + #[test] + fn ast_golden_replay() { + let fixture = include_str!("../../tests/fixtures/ast_golden.json"); + let root: serde_json::Value = serde_json::from_str(fixture).unwrap(); + let cases = root + .get("cases") + .and_then(serde_json::Value::as_array) + .unwrap(); + assert_eq!(cases.len(), 18); + for case in cases { + let path = case["path"].as_str().unwrap(); + let content = case["content"].as_str().unwrap(); + let recorded = case["points"].as_array().unwrap(); + // Upstream records UTF-16 positions — the public API + // must match verbatim on every file. + let got = ast_break_points(content, path); + assert_eq!(got.len(), recorded.len(), "{path} count"); + for (g, r) in got.iter().zip(recorded) { + assert_eq!( + g.pos, + usize::try_from(r["pos"].as_u64().unwrap()).unwrap(), + "{path} pos" + ); + assert_eq!( + g.score, + u32::try_from(r["score"].as_u64().unwrap()).unwrap(), + "{path} score" + ); + assert_eq!(g.kind.as_ref(), r["type"].as_str().unwrap(), "{path} type"); + } + // ASCII files are also byte-identical to the raw view. + if content.is_ascii() { + let raw = break_points_utf8(content, path); + assert!( + raw.iter().zip(&got).all(|(a, b)| a.pos == b.pos), + "{path} byte-vs-utf16" + ); + } + } + } +} diff --git a/crates/qmd/tests/fixtures/ast_golden.json b/crates/qmd/tests/fixtures/ast_golden.json new file mode 100644 index 0000000..da6cd7d --- /dev/null +++ b/crates/qmd/tests/fixtures/ast_golden.json @@ -0,0 +1,490 @@ +{ + "generated": "2026-10-29", + "upstream": "tobi/qmd src/ast.ts getASTBreakPoints (pos = UTF-16 code-unit index)", + "cases": [ + { + "path": "src/auth.ts", + "content": "import { Database } from './db';\nimport type { User } from './types';\n\ninterface AuthConfig {\n secret: string;\n ttl: number;\n}\n\ntype UserId = string;\n\nexport class AuthService {\n constructor(private db: Database) {}\n\n async authenticate(user: User, token: string): Promise {\n const session = await this.db.findSession(token);\n return session?.userId === user.id;\n }\n\n validateToken(token: string): boolean {\n return token.length === 64;\n }\n}\n\nexport function hashPassword(password: string): string {\n return crypto.createHash('sha256').update(password).digest('hex');\n}\n", + "points": [ + { + "pos": 0, + "score": 60, + "type": "ast:import" + }, + { + "pos": 33, + "score": 60, + "type": "ast:import" + }, + { + "pos": 71, + "score": 100, + "type": "ast:iface" + }, + { + "pos": 130, + "score": 80, + "type": "ast:type" + }, + { + "pos": 153, + "score": 90, + "type": "ast:export" + }, + { + "pos": 160, + "score": 100, + "type": "ast:class" + }, + { + "pos": 182, + "score": 90, + "type": "ast:method" + }, + { + "pos": 222, + "score": 90, + "type": "ast:method" + }, + { + "pos": 389, + "score": 90, + "type": "ast:method" + }, + { + "pos": 468, + "score": 90, + "type": "ast:export" + }, + { + "pos": 475, + "score": 90, + "type": "ast:func" + } + ] + }, + { + "path": "mod/utils.mts", + "content": "import { join } from 'path';\n\nenum Kind { A, B }\n\nexport const parse = (s: string): string => s.trim();\n\nfunction helper() { return 1; }\n", + "points": [ + { + "pos": 0, + "score": 60, + "type": "ast:import" + }, + { + "pos": 30, + "score": 80, + "type": "ast:enum" + }, + { + "pos": 50, + "score": 90, + "type": "ast:export" + }, + { + "pos": 57, + "score": 90, + "type": "ast:func" + }, + { + "pos": 105, + "score": 90, + "type": "ast:func" + } + ] + }, + { + "path": "conf/settings.cts", + "content": "import fs from 'fs';\n\nexport const load = function(path: string) { return fs.readFileSync(path); };\n\nclass Cache { get() { return null; } }\n", + "points": [ + { + "pos": 0, + "score": 60, + "type": "ast:import" + }, + { + "pos": 22, + "score": 90, + "type": "ast:export" + }, + { + "pos": 29, + "score": 90, + "type": "ast:func" + }, + { + "pos": 101, + "score": 100, + "type": "ast:class" + }, + { + "pos": 115, + "score": 90, + "type": "ast:method" + } + ] + }, + { + "path": "src/App.tsx", + "content": "import React from 'react';\n\ninterface Props { name: string; }\n\ntype Size = 'sm' | 'lg';\n\nenum Variant { A, B }\n\nexport class Widget {\n render() { return
; }\n}\n\nexport const Label = (p: Props) => {p.name};\n", + "points": [ + { + "pos": 0, + "score": 60, + "type": "ast:import" + }, + { + "pos": 28, + "score": 100, + "type": "ast:iface" + }, + { + "pos": 63, + "score": 80, + "type": "ast:type" + }, + { + "pos": 89, + "score": 80, + "type": "ast:enum" + }, + { + "pos": 112, + "score": 90, + "type": "ast:export" + }, + { + "pos": 119, + "score": 100, + "type": "ast:class" + }, + { + "pos": 136, + "score": 90, + "type": "ast:method" + }, + { + "pos": 167, + "score": 90, + "type": "ast:export" + }, + { + "pos": 174, + "score": 90, + "type": "ast:func" + } + ] + }, + { + "path": "src/View.jsx", + "content": "import { useState } from 'react';\n\nexport default function View() {\n return
;\n}\n\nconst Btn = () =>