feat(collections): opt into hidden directories

CHANGELOG.md

@@ -13,6 +13,7 @@
 
 ### Added
 
+- Per-collection hidden directory opt-in through `qmd collection include-dot-dirs` and the SDK's `includeDotDirs` setting. Collection masks and exclusions remain effective, and hidden files stay excluded. Settings persist in self-contained SQLite stores.
 - Added Oxlint lint fence.
 - Document metadata and metadata filtering. Markdown documents can opt into typed metadata through a namespaced frontmatter block (`qmd.metadata` with strings, numbers, booleans, or flat homogeneous arrays), and every search surface — CLI `search`/`vsearch`/`query` via `--filter <json>`, the SDK's `filter` option on `search()`/`searchLex()`/`searchVector()`, the MCP `query` tool, and HTTP `POST /query` and `/search` — accepts one shared recursive filter AST discriminated by `operator`: `and`/`or`/`not` logical groups, `eq`/`ne`/`gt`/`gte`/`lt`/`lte` comparisons, `in`/`nin`/`all` membership, `contains`/`prefix`/`suffix` text matching, `type` for the stored type of a key's values, and `exists` presence, with an optional `caseInsensitive` flag on any condition whose value is a string (ASCII folding). Every returned result satisfies the filter (applied before RRF fusion and reranking); like collection filtering, highly selective filters remain best-effort for top-K completeness. Frontmatter stays ordinary searchable content — no chunking, embedding, snippet, or line-number changes — and documents without `qmd.metadata` behave exactly as before. JSON/SDK/MCP/HTTP results now include each document's indexed metadata, and `qmd status` reports how many documents still need metadata extraction (a normal `qmd update` backfills existing indexes).
 - Metadata discovery. Filtering is only useful when the caller knows what to filter on, so every surface now reports the metadata keys, types, and value counts already in the index — read from the existing metadata tables, with no re-indexing. The CLI adds `qmd collection metadata [name...]` with `--filter <json>` to count only documents matching a filter (same AST as search) and `--match <json>` to select which metadata entries are reported (the same AST evaluated per entry, where a condition's `field` is the entry's `key` or `value`, so one grammar covers a single key, a family of keys, reverse lookup of a value, typed thresholds, and any `and`/`or`/`not` composition), plus `--key-limit`/`--key-offset`/`--all-keys` and `--value-limit`/`--value-offset`/`--all-values` to page both windows, and `--sort count|value` and `--min-count <n>` to order and trim values. Every windowed list reports its exact remainder, counts are documents rather than values, numbers report min/median/max, keys whose documents disagree on type are split per type with their own counts, and with `--filter` every coverage count is measured against the documents the filter admits. `qmd collection list` names each collection's top keys, `qmd collection show` details them with a value preview, and `qmd status` summarizes coverage. The SDK adds `listMetadata(options)` (the same options, spelled `keyLimit`, `valueLimit`, and so on, with `MetadataOptionError` for a value outside its domain and `MetadataBindingBudgetError` for a filter and match that together bind too many SQL parameters), the MCP server adds a `metadata` tool and lists each collection's most covered keys in `status`, and the HTTP server adds `POST /metadata`. `MetadataFilter` and `MetadataMatch` are both `MetadataPredicate<Condition>`, one recursive grammar over the conditions a document or a metadata entry admits.

CLAUDE.md

@@ -51,6 +51,10 @@ qmd collection rename mynotes my-notes
 # Show collection details, including the top metadata keys
 qmd collection show mynotes
 
+# Opt into .claude/ and .agents/ directories; omit names to clear
+qmd collection include-dot-dirs mynotes claude agents
+qmd update
+
 # Discover metadata keys and values to filter on
 qmd collection metadata mynotes
 qmd collection metadata mynotes --match '{"field":"key","operator":"eq","value":"topics"}'

README.md

@@ -670,8 +670,18 @@ qmd collection exclude notes
 # Run a command before every `qmd update` (e.g. git pull); empty arg clears it
 qmd collection update-cmd notes 'git pull --rebase'
 qmd collection update-cmd notes
+
+# Include .claude/ and .agents/ directories, then apply the setting
+qmd collection include-dot-dirs myproject claude agents
+qmd update
+
+# Clear the setting; the next update removes those hidden documents
+qmd collection include-dot-dirs myproject
+qmd update
 ```
 
+Hidden directories are excluded by default. `include-dot-dirs` opts a collection into literal directory names at any depth; a leading dot is optional in the CLI. Hidden files remain excluded, and collection masks, ignore patterns, and built-in exclusions still apply. The SDK accepts the same names without leading dots through `Collection.includeDotDirs` or `addCollection(..., { includeDotDirs: ["claude"] })`.
+
 ### Generate Vector Embeddings
 
 ```sh
@@ -771,6 +781,7 @@ collections:
       - "**/drafts/**"
     update: "git pull --rebase"  # bash command run before each `qmd update`
     includeByDefault: true       # include in unscoped queries (default: true)
+    includeDotDirs: ["claude"]    # index .claude/ directories (default: none)
     context:                     # path prefix → description; longest match wins
       "/": "Personal notes and ideas"
       "/work": "Work-related notes"
@@ -786,11 +797,10 @@ collections:
 | `collections.<name>.ignore` | per-collection | Glob patterns excluded from indexing — useful to stop nested collections double-indexing. **YAML-only — no CLI command sets this.** Additive with QMD's built-in exclusions (`node_modules`, `.git`, `.cache`, `vendor`, `dist`, `build`), which you cannot un-ignore. |
 | `collections.<name>.update` | per-collection | Bash command run before `qmd update` re-indexes this collection. Set via `qmd collection update-cmd`. |
 | `collections.<name>.includeByDefault` | per-collection | Whether unscoped queries search it. Toggle with `qmd collection include`/`exclude`. Default `true`. |
+| `collections.<name>.includeDotDirs` | per-collection | Hidden directory names to index, without leading dots or glob syntax. Set via `qmd collection include-dot-dirs`. Default `[]`; hidden files stay excluded. |
 | `collections.<name>.context` | per-collection | Path-prefix → description map; the most specific (longest) matching prefix wins. Set via `qmd context add`. |
 
-> **Note:** Editing `index.yml` changes which directories and models QMD *uses*,
-> but does not re-index on its own. Run `qmd update` after changing `path`,
-> `pattern`, or `ignore`, and `qmd embed` after changing `models.embed`.
+> **Note:** Editing `index.yml` changes which directories and models QMD *uses*, but does not re-index on its own. Run `qmd update` after changing `path`, `pattern`, `ignore`, or `includeDotDirs`, and `qmd embed` after changing `models.embed`.
 
 #### Automatic update commands
 

skills/qmd/SKILL.md

@@ -322,6 +322,8 @@ qmd update
 qmd embed
 ```
 
+Hidden directories are excluded by default. When the user authorizes indexing them, opt in per collection with `qmd collection include-dot-dirs notes claude agents`, then run `qmd update`. Omit the names to clear the setting and run `qmd update` to remove those documents. Masks and ignore patterns still apply; hidden files remain excluded.
+
 Health and diagnostics:
 
 ```bash

src/cli/qmd.ts

@@ -1,6 +1,5 @@
 import { isBun, openDatabase } from "../db.js";
 import type { Database, SQLiteValue } from "../db.js";
-import fastGlob from "fast-glob";
 import { spawn as nodeSpawn } from "child_process";
 import { isQmdMcpPid, mcpDaemonStateFiles } from "./mcp-pid.js";
 import { embedLockPathForDb, tryAcquireEmbedLock, EMBED_LOCK_BUSY_MESSAGE } from "./embed-lock.js";
@@ -86,6 +85,7 @@ import {
   reindexCollection,
   scanWriteBatch,
   REINDEX_MAX_FILE_SIZE,
+  globCollectionFiles,
   generateEmbeddings,
   maybeAdoptLegacyEmbeddingFingerprint,
   syncConfigToDb,
@@ -998,6 +998,7 @@ async function updateCollections(): Promise<void> {
 
     const result = await reindexCollection(storeInstance, col.pwd, col.glob_pattern, col.name, {
       ignorePatterns: yamlCol?.ignore,
+      includeDotDirs: yamlCol?.includeDotDirs,
       onProgress: (info) => {
         progress.set((info.current / info.total) * 100);
         const elapsed = (Date.now() - startTime) / 1000;
@@ -1838,6 +1839,9 @@ function collectionList(): void {
     if (yamlColl?.ignore?.length) {
       console.log(`  ${c.dim}Ignore:${c.reset}   ${yamlColl.ignore.join(', ')}`);
     }
+    if (yamlColl?.includeDotDirs?.length) {
+      console.log(`  ${c.dim}Include-dot-dirs:${c.reset} ${yamlColl.includeDotDirs.map(d => `.${d}`).join(', ')}`);
+    }
     console.log(`  ${c.dim}Files:${c.reset}    ${coll.active_count}`);
     const metadataOverview = metadataOverviews.get(coll.name);
     if (metadataOverview) {
@@ -1917,7 +1921,7 @@ async function collectionAdd(pwd: string, globPattern: string, name?: string): P
   // Create the collection and index files
   console.log(`Creating collection '${collName}'...`);
   const newColl = getCollectionFromYaml(collName);
-  await indexFiles(pwd, globPattern, collName, false, newColl?.ignore);
+  await indexFiles(pwd, globPattern, collName, false, newColl?.ignore, newColl?.includeDotDirs);
   console.log(`${c.green}✓${c.reset} Collection '${collName}' created successfully`);
 }
 
@@ -2092,7 +2096,7 @@ function parseNonNegativeInteger(raw: unknown, flag: string): number {
   return parsed;
 }
 
-async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GLOB, collectionName?: string, suppressEmbedNotice: boolean = false, ignorePatterns?: string[]): Promise<void> {
+async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GLOB, collectionName?: string, suppressEmbedNotice: boolean = false, ignorePatterns?: string[], includeDotDirs?: string[]): Promise<void> {
   const db = getDb();
   const resolvedPwd = pwd || getPwd();
   const now = new Date().toISOString();
@@ -2113,18 +2117,7 @@ async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GLOB, coll
     ...excludeDirs.map(d => `**/${d}/**`),
     ...(ignorePatterns || []),
   ];
-  const allFiles: string[] = await fastGlob(splitGlobMask(globPattern), {
-    cwd: resolvedPwd,
-    onlyFiles: true,
-    followSymbolicLinks: false,
-    dot: false,
-    ignore: allIgnore,
-  });
-  // Filter hidden files/folders (dot: false handles top-level but not nested)
-  const files = allFiles.filter(file => {
-    const parts = file.split("/");
-    return !parts.some(part => part.startsWith("."));
-  });
+  const files = await globCollectionFiles(globPattern, { cwd: resolvedPwd, ignore: allIgnore, includeDotDirs });
 
   const total = files.length;
   const hasNoFiles = total === 0;
@@ -4836,6 +4829,35 @@ if (isMain) {
           break;
         }
 
+        case "include-dot-dirs": {
+          const name = cli.args[1];
+          const dirs = cli.args.slice(2).map(d => d.replace(/^\./, ""));
+          if (!name) {
+            console.error("Usage: qmd collection include-dot-dirs <name> [dir ...]");
+            console.error("  Crawl these otherwise-hidden directory names during indexing (e.g. 'claude' for .claude/)");
+            console.error("  Omit dirs to clear the list (hidden directories stay excluded by default)");
+            process.exit(1);
+          }
+          const { updateCollectionSettings, getCollection } = await import("../collections.js");
+          const col = getCollection(name);
+          if (!col) {
+            console.error(`Collection not found: ${name}`);
+            process.exit(1);
+          }
+          try {
+            updateCollectionSettings(name, { includeDotDirs: dirs.length ? dirs : null });
+          } catch (error) {
+            console.error(error instanceof Error ? error.message : String(error));
+            process.exit(1);
+          }
+          if (dirs.length) {
+            console.log(`✓ Collection '${name}' now includes dot-directories: ${dirs.map(d => `.${d}`).join(', ')}`);
+          } else {
+            console.log(`✓ Cleared dot-directory allow-list for '${name}'`);
+          }
+          break;
+        }
+
         case "include":
         case "exclude": {
           const name = cli.args[1];
@@ -4873,6 +4895,12 @@ if (isMain) {
           console.log(`  Path:     ${col.path}`);
           console.log(`  Pattern:  ${col.pattern}`);
           console.log(`  Include:  ${col.includeByDefault !== false ? 'yes (default)' : 'no'}`);
+          if (col.ignore?.length) {
+            console.log(`  Ignore:   ${col.ignore.join(', ')}`);
+          }
+          if (col.includeDotDirs?.length) {
+            console.log(`  Include-dot-dirs: ${col.includeDotDirs.map(d => `.${d}`).join(', ')}`);
+          }
           if (col.update) {
             console.log(`  Update:   ${col.update}`);
           }
@@ -4911,13 +4939,17 @@ if (isMain) {
           console.log("    --value-limit <n>       Values per key (default 10), --value-offset <n> pages, --all-values removes the window");
           console.log("    --sort count|value      Value order (default count), --min-count <n> drops the tail");
           console.log("  update-cmd <name> [cmd]   Set pre-update command (e.g., 'git pull')");
+          console.log("  include-dot-dirs <name> [dir ...]");
+          console.log("                            Crawl these hidden directory names (e.g. 'claude' for .claude/)");
+          console.log("                            despite the default dotdirectory exclusion; omit to clear");
           console.log("  include <name>            Include in default queries");
           console.log("  exclude <name>            Exclude from default queries");
           console.log("");
           console.log("Examples:");
           console.log("  qmd collection add ~/notes --name notes");
           console.log("  qmd collection add ~/notes --name notes --mask 'a.md,journals/*.md'");
           console.log("  qmd collection update-cmd brain 'git pull'");
+          console.log("  qmd collection include-dot-dirs notes claude");
           console.log("  qmd collection exclude archive");
           console.log("  qmd collection metadata notes --match '{\"field\":\"key\",\"operator\":\"eq\",\"value\":\"topics\"}'");
           console.log("  qmd collection metadata notes --match '{\"field\":\"value\",\"operator\":\"eq\",\"value\":\"docs-team\"}'");

src/collections.ts

@@ -31,6 +31,18 @@ export interface Collection {
   context?: ContextMap;      // Optional context definitions
   update?: string;           // Optional bash command to run during qmd update
   includeByDefault?: boolean; // Include in queries by default (default: true)
+  includeDotDirs?: string[]; // Hidden directory names without leading dots (default: none)
+}
+
+/** Validate literal directory names before using them in filesystem globs. */
+export function parseIncludeDotDirs(value: unknown): string[] {
+  if (value === undefined) return [];
+  if (!Array.isArray(value) || Array.from(value).some(name =>
+    typeof name !== "string" || !name || name.startsWith(".") || /[\0/\\*?[\]{}()!|]/.test(name)
+  )) {
+    throw new Error("includeDotDirs must be an array of literal directory names without leading dots, path separators, or glob characters");
+  }
+  return [...new Set(value as string[])];
 }
 
 /**
@@ -183,6 +195,9 @@ function ensureConfigDir(): void {
 export function loadConfig(): CollectionConfig {
   // SDK inline config mode
   if (configSource.type === 'inline') {
+    for (const collection of Object.values(configSource.config.collections)) {
+      parseIncludeDotDirs(collection.includeDotDirs);
+    }
     return configSource.config;
   }
 
@@ -202,6 +217,9 @@ export function loadConfig(): CollectionConfig {
       config.collections = {};
     }
 
+    for (const collection of Object.values(config.collections)) {
+      parseIncludeDotDirs(collection.includeDotDirs);
+    }
     return config;
   } catch (error) {
     throw new Error(`Failed to parse ${configPath}: ${error}`);
@@ -214,6 +232,9 @@ export function loadConfig(): CollectionConfig {
  * - File-based: writes to YAML file (default ~/.config/qmd/index.yml)
  */
 export function saveConfig(config: CollectionConfig): void {
+  for (const collection of Object.values(config.collections)) {
+    parseIncludeDotDirs(collection.includeDotDirs);
+  }
   // SDK inline config mode: update in place, no file I/O
   if (configSource.type === 'inline') {
     configSource.config = config;
@@ -282,8 +303,10 @@ export function getDefaultCollectionNames(): string[] {
  */
 export function updateCollectionSettings(
   name: string,
-  settings: { update?: string | null; includeByDefault?: boolean }
+  settings: { update?: string | null; includeByDefault?: boolean; includeDotDirs?: string[] | null }
 ): boolean {
+  const includeDotDirs = settings.includeDotDirs === null
+    ? [] : parseIncludeDotDirs(settings.includeDotDirs);
   const config = loadConfig();
   const collection = config.collections[name];
   if (!collection) return false;
@@ -305,6 +328,14 @@ export function updateCollectionSettings(
     }
   }
 
+  if (settings.includeDotDirs !== undefined) {
+    if (includeDotDirs.length === 0) {
+      delete collection.includeDotDirs;
+    } else {
+      collection.includeDotDirs = includeDotDirs;
+    }
+  }
+
   saveConfig(config);
   return true;
 }

src/index.ts

@@ -105,6 +105,7 @@ import {
 import {
   setConfigSource,
   loadConfig,
+  updateCollectionSettings,
   addCollection as collectionsAddCollection,
   removeCollection as collectionsRemoveCollection,
   renameCollection as collectionsRenameCollection,
@@ -334,7 +335,7 @@ export interface QMDStore {
   // ── Collection Management ───────────────────────────────────────────
 
   /** Add or update a collection */
-  addCollection(name: string, opts: { path: string; pattern?: string; ignore?: string[] }): Promise<void>;
+  addCollection(name: string, opts: { path: string; pattern?: string; ignore?: string[]; includeDotDirs?: string[] }): Promise<void>;
 
   /** Remove a collection */
   removeCollection(name: string): Promise<boolean>;
@@ -548,9 +549,17 @@ export async function createStore(options: StoreOptions): Promise<QMDStore> {
 
     // Collection Management — write to SQLite + write-through to YAML/inline if configured
     addCollection: async (name, opts) => {
-      upsertStoreCollection(db, name, { path: opts.path, pattern: opts.pattern, ignore: opts.ignore });
+      upsertStoreCollection(db, name, {
+        path: opts.path,
+        pattern: opts.pattern,
+        ignore: opts.ignore,
+        includeDotDirs: opts.includeDotDirs,
+      });
       if (hasYamlConfig || options.config) {
         collectionsAddCollection(name, opts.path, opts.pattern);
+        if (opts.includeDotDirs !== undefined) {
+          updateCollectionSettings(name, { includeDotDirs: opts.includeDotDirs });
+        }
       }
     },
     removeCollection: async (name) => {
@@ -616,6 +625,7 @@ export async function createStore(options: StoreOptions): Promise<QMDStore> {
       for (const col of filtered) {
         const result = await reindexCollection(internal, col.path, col.pattern || "**/*.md", col.name, {
           ignorePatterns: col.ignore,
+          includeDotDirs: col.includeDotDirs,
           onProgress: updateOpts?.onProgress
             ? (info) => updateOpts.onProgress!({ collection: col.name, ...info })
             : undefined,

src/mcp/server.ts

@@ -206,6 +206,7 @@ async function buildInstructions(store: QMDStore): Promise<string> {
   // --- Non-obvious things that prevent mistakes ---
   lines.push("");
   lines.push("Tips:");
+  lines.push("  - Hidden directories are excluded by default. If the user authorizes indexing .claude/ documents, run `qmd collection include-dot-dirs <collection> claude` in the CLI, then `qmd update`. Hidden files, collection masks, and ignore exclusions still apply.");
   lines.push("  - File paths in results are relative to their collection.");
   lines.push("  - Use `minScore: 0.5` to filter low-confidence results.");
   lines.push("  - Results include a `context` field describing the content type.");